[SERVER-24711] ASIO connections that have already timed out before setup should not error Created: 22/Jun/16  Updated: 08/Jan/24  Resolved: 27/Jun/16

Status: Closed
Project: Core Server
Component/s: Concurrency, Networking, Stability
Affects Version/s: 3.2.7
Fix Version/s: 3.2.8, 3.3.9

Type: Bug Priority: Critical - P2
Reporter: Edik Mkoyan Assignee: Matt Cotter
Resolution: Done Votes: 1
Labels: code-and-test
Remaining Estimate: Not Specified
Time Spent: Not Specified
Original Estimate: Not Specified

Issue Links:
Depends
depends on SERVER-24058 Connection pool asio doesn't honor se... Closed
Duplicate
is duplicated by SERVER-24969 Abort on loss of network in replicaset Closed
is duplicated by SERVER-24834 Two members of replica set threw erro... Closed
is duplicated by SERVER-24874 Mongod Crash on flakey network Closed
Related
Backwards Compatibility: Fully Compatible
Operating System: ALL
Backport Completed:
Steps To Reproduce:

It happened by itself, probably network segmentation.

Sprint: Platforms 16 (06/24/16), Platforms 17 (07/15/16)
Participants:
Case:
Linked BF Score: 0

 Description   

I have a multi site setup of MongoDB 3.2.7 in four locations, today, two config servers crashed with the following log.

016-06-21T18:46:17.059+0400 I REPL     [ReplicationExecutor] Error in heartbeat request to mcfg01:27019; ExceededTimeLimit: Couldn't get a connection within the time limit
2016-06-21T18:46:20.447+0400 I ACCESS   [conn1] Successfully authenticated as principal root on admin
2016-06-21T18:46:27.059+0400 I -        [NetworkInterfaceASIO-Replication-0] Invariant failure _connection.is_initialized() src/mongo/executor/network_interface_asio_operation.cpp 142
2016-06-21T18:46:27.059+0400 I -        [NetworkInterfaceASIO-Replication-0] 
 
***aborting after invariant() failure
 
 
2016-06-21T18:46:27.421+0400 F -        [NetworkInterfaceASIO-Replication-0] Got signal: 6 (Aborted).
 
 0x1319fa2 0x13190f9 0x1319902 0x7f537c2a8340 0x7f537bf09cc9 0x7f537bf0d0d8 0x12a380b 0x10e766d 0x10c6643 0x10c763c 0x10c7ba7 0x10c82d9 0x1335cb1 0x1335ed1 0x133a06f 0x10d36b5 0x1b34160 0x7f537c2a0182 0x7f537bfcd47d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"F19FA2","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"F190F9"},{"b":"400000","o":"F19902"},{"b":"7F537C298000","o":"10340"},{"b":"7F537BED3000","o":"36CC9","s":"gsignal"},{"b":"7F537BED3000","o":"3A0D8","s":"abort"},{"b":"400000","o":"EA380B","s":"_ZN5mongo15invariantFailedEPKcS1_j"},{"b":"400000","o":"CE766D"},{"b":"400000","o":"CC6643"},{"b":"400000","o":"CC763C"},{"b":"400000","o":"CC7BA7"},{"b":"400000","o":"CC82D9"},{"b":"400000","o":"F35CB1","s":"_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code"},{"b":"400000","o":"F35ED1","s":"_ZN4asio6detail9scheduler3runERSt10error_code"},{"b":"400000","o":"F3A06F","s":"_ZN4asio10io_service3runEv"},{"b":"400000","o":"CD36B5"},{"b":"400000","o":"1734160","s":"execute_native_thread_routine"},{"b":"7F537C298000","o":"8182"},{"b":"7F537BED3000","o":"FA47D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.2.7", "gitVersion" : "4249c1d2b5999ebbf1fdf3bc0e0e3b3ff5c0aaf2", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.16.0-60-generic", "version" : "#80~14.04.1-Ubuntu SMP Wed Jan 20 13:37:48 UTC 2016", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "A8DC02B241532EEAD31FFE2633EE8C8E565C1F1A" }, { "b" : "7FFE735E9000", "elfType" : 3, "buildId" : "E909F3BEA8B99B483153329FF4F90B288A068655" }, { "b" : "7F537D1B9000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "D08DD65F97859C71BB2CBBF1043BD968EFE18AAD" }, { "b" : "7F537CDDE000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "F86FA9FB4ECEB4E06B40DBDF761A4172B70A4229" }, { "b" : "7F537CBD6000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "B376100CAB1EAC4E5DE066EACFC282BF7C0B54F3" }, { "b" : "7F537C9D2000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "67699FFDA9FD2A552032E0652A242E82D65AA10D" }, { "b" : "7F537C6CC000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "EF3F6DFFA1FBE48436EC6F45CD3AABA157064BB4" }, { "b" : "7F537C4B6000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "36311B4457710AE5578C4BF00791DED7359DBB92" }, { "b" : "7F537C298000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "AF06068681750736E0524DF17D5A86CB2C3F765C" }, { "b" : "7F537BED3000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "5382058B69031CAA9B9996C11061CD164C9398FF" }, { "b" : "7F537D418000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "2A816C3EBBA4E12813FBD34B06FBD25BC892A67F" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x1319fa2]
 mongod(+0xF190F9) [0x13190f9]
 mongod(+0xF19902) [0x1319902]
 libpthread.so.0(+0x10340) [0x7f537c2a8340]
 libc.so.6(gsignal+0x39) [0x7f537bf09cc9]
 libc.so.6(abort+0x148) [0x7f537bf0d0d8]
 mongod(_ZN5mongo15invariantFailedEPKcS1_j+0xCB) [0x12a380b]
 mongod(+0xCE766D) [0x10e766d]
 mongod(+0xCC6643) [0x10c6643]
 mongod(+0xCC763C) [0x10c763c]
 mongod(+0xCC7BA7) [0x10c7ba7]
 mongod(+0xCC82D9) [0x10c82d9]
 mongod(_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code+0x2F1) [0x1335cb1]
 mongod(_ZN4asio6detail9scheduler3runERSt10error_code+0xC1) [0x1335ed1]
 mongod(_ZN4asio10io_service3runEv+0x2F) [0x133a06f]
 mongod(+0xCD36B5) [0x10d36b5]
 mongod(execute_native_thread_routine+0x20) [0x1b34160]
 libpthread.so.0(+0x8182) [0x7f537c2a0182]
 libc.so.6(clone+0x6D) [0x7f537bfcd47d]
-----  END BACKTRACE  -----



 Comments   
Comment by Mira Carey [ 21/Jul/16 ]

zyd_com,

ASIO is meant to manage it's own heartbeats at the application layer, but unfortunately that feature's been broken in much of 3.2. A combination of SERVER-24058, which prevented new connections from being established, and SERVER-25006, which prevented the heartbeats from going out, led to the kind of false networking partitions you've seen. As of 3.2.7, you should no longer see "Couldn't get a connection within the time limit" and after 3.2.9 is released regular heartbeating should further reduce the number of transient networking errors that bubble up.

Comment by Zhang Youdong [ 21/Jul/16 ]

Ramon Fernandez I have figured out what's going on in my case.

node1、node2、node3 consturct a replica set,it works well at the very begining, node3 crashes by accident , after restart, node1 cannot send heartbeat message to node3, the error message is "Couldn't get a connection within the time limit".

From netstat ouput, I found node1 have estabalished a connection with node3, but node3 didn't have this connection, because this connection is established before node3 crash, and the tcp keepalive timer is off, so this connection wasn't closed when node3 crash.

tcp        0      0 node1:58347          node3:3015           ESTABLISHED off (0.00/0/0)

I use libnet tool to send some data to this link,the connection will closed immediately,node1 reconnect to node3,everything is ok.

So I think there must be something run which ASIO framework, but I cannot find it out by reading the source code now, I want to know if SERVER-24058 solve this case?

And I am curious why ASIO don't enable tcp keepalive?

Comment by Ramon Fernandez Marina [ 21/Jul/16 ]

zyd_com, there's been some improvements since 3.2.3 so I'd recommend you upgrade to 3.2.8 and open new tickets if you still experience issues. SERVER-24058 may be at play here, not sure, but that has been fixed in 3.2.7.

Comment by Zhang Youdong [ 21/Jul/16 ]

I have encounter the similar problem except the node didn't crash when using 3.2.3

rs.status() from node1

mgset-1095589:PRIMARY> rs.status()
{
	"set" : "mgset-1095589",
	"date" : ISODate("2016-07-21T06:12:06.413Z"),
	"myState" : 1,
	"term" : NumberLong(-1),
	"heartbeatIntervalMillis" : NumberLong(2000),
	"members" : [
		{
			"_id" : 1,
			"name" : "node1:3017",
			"health" : 1,
			"state" : 1,
			"stateStr" : "PRIMARY",
			"uptime" : 1198471,
			"optime" : Timestamp(1469081526, 462),
			"optimeDate" : ISODate("2016-07-21T06:12:06Z"),
			"electionTime" : Timestamp(1469040106, 1),
			"electionDate" : ISODate("2016-07-20T18:41:46Z"),
			"configVersion" : 60046,
			"self" : true
		},
		{
			"_id" : 2,
			"name" : "node2:3010",
			"health" : 1,
			"state" : 2,
			"stateStr" : "SECONDARY",
			"uptime" : 1198469,
			"optime" : Timestamp(1469077724, 52),
			"optimeDate" : ISODate("2016-07-21T05:08:44Z"),
			"lastHeartbeat" : ISODate("2016-07-21T06:12:06.294Z"),
			"lastHeartbeatRecv" : ISODate("2016-07-21T06:12:04.983Z"),
			"pingMs" : NumberLong(0),
			"syncingTo" : "10.151.14.96:3017",
			"configVersion" : 60046
		},
		{
			"_id" : 3,
			"name" : "node3:3015",
			"health" : 0,
			"state" : 8,
			"stateStr" : "(not reachable/healthy)",
			"uptime" : 0,
			"optime" : Timestamp(0, 0),
			"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
			"lastHeartbeat" : ISODate("2016-07-21T06:11:56.416Z"),
			"lastHeartbeatRecv" : ISODate("2016-07-21T06:12:05.786Z"),
			"pingMs" : NumberLong(0),
			"lastHeartbeatMessage" : "Couldn't get a connection within the time limit",
			"configVersion" : -1
		}
	],
	"ok" : 1
}

rs.status() from node3

 
mgset-1095589:SECONDARY> rs.status()
{
	"set" : "mgset-1095589",
	"date" : ISODate("2016-07-21T08:12:28.627Z"),
	"myState" : 2,
	"term" : NumberLong(-1),
	"heartbeatIntervalMillis" : NumberLong(2000),
	"members" : [
		{
			"_id" : 1,
			"name" : "node1:3017",
			"health" : 1,
			"state" : 1,
			"stateStr" : "PRIMARY",
			"uptime" : 378,
			"optime" : Timestamp(1469088746, 765),
			"optimeDate" : ISODate("2016-07-21T08:12:26Z"),
			"lastHeartbeat" : ISODate("2016-07-21T08:12:26.644Z"),
			"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
			"pingMs" : NumberLong(0),
			"electionTime" : Timestamp(1469040106, 1),
			"electionDate" : ISODate("2016-07-20T18:41:46Z"),
			"configVersion" : 60046
		},
		{
			"_id" : 2,
			"name" : "node2:3010",
			"health" : 1,
			"state" : 2,
			"stateStr" : "SECONDARY",
			"uptime" : 378,
			"optime" : Timestamp(1469087563, 228),
			"optimeDate" : ISODate("2016-07-21T07:52:43Z"),
			"lastHeartbeat" : ISODate("2016-07-21T08:12:26.645Z"),
			"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
			"pingMs" : NumberLong(0),
			"syncingTo" : "10.151.14.96:3017",
			"configVersion" : 60046
		},
		{
			"_id" : 3,
			"name" : "node3:3015",
			"health" : 1,
			"state" : 2,
			"stateStr" : "SECONDARY",
			"uptime" : 390,
			"optime" : Timestamp(1469087452, 326),
			"optimeDate" : ISODate("2016-07-21T07:50:52Z"),
			"syncingTo" : "10.151.14.96:3017",
			"configVersion" : 60046,
			"self" : true
		}
	],
	"ok" : 1
}

From the netstat output from node1, node1 have established a connection with node3.
From the netstat output from node3, node1 doesn't established a connection with node3.

Is this the same problem?

Comment by Ramon Fernandez Marina [ 08/Jul/16 ]

All,

MongoDB release candidate 3.2.8-rc1 is now available for download, and includes a fix for this issue. The MongoDB 3.2.8 release is scheduled for next week.

Regards,
Ramón.

Comment by Kelsey Schubert [ 01/Jul/16 ]

Hi tch@virtall.com,

Thank you for the report and sorry that you have encountered this issue. A release candidate for MongoDB 3.2.8, which will include this fix, is currently scheduled for next week.

Kind regards,
Thomas

Comment by Tomasz Chmielewski [ 01/Jul/16 ]

Saw a similar crash today, with 3.2.7 on Ubuntu 14.04:

2016-07-01T09:13:45.690+0000 I -        [NetworkInterfaceASIO-Replication-0] Invariant failure _connection.is_initialized() src/mongo/executor/network_interface_asio_operation.cpp 142
2016-07-01T09:13:45.690+0000 I -        [NetworkInterfaceASIO-Replication-0] 
 
***aborting after invariant() failure
 
 
2016-07-01T09:13:45.703+0000 I NETWORK  [initandlisten] connection accepted from 10.0.0.139:59905 #56956 (14 connections now open)
2016-07-01T09:13:45.706+0000 I NETWORK  [initandlisten] connection accepted from 10.0.0.117:34130 #56957 (15 connections now open)
2016-07-01T09:13:45.772+0000 I NETWORK  [initandlisten] connection accepted from 10.0.0.116:37864 #56958 (16 connections now open)
2016-07-01T09:13:45.860+0000 F -        [NetworkInterfaceASIO-Replication-0] Got signal: 6 (Aborted).
 
 0x1319fa2 0x13190f9 0x1319902 0x7f9128ff6330 0x7f9128c57c37 0x7f9128c5b028 0x12a380b 0x10e766d 0x10c6643 0x10c763c 0x10c7ba7 0x10c82d9 0x1335cb1 0x1335ed1 0x133a06f 0x10d36b5 0x1b34160 0x7f9128fee184 0x7f9128d1b37d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"F19FA2","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"F190F9"},{"b":"400000","o":"F19902"},{"b":"7F9128FE6000","o":"10330"},{"b":"7F9128C21000","o":"36C37","s":"gsignal"},{"b":"7F9128C21000","o":"3A028","s":"abort"},{"b":"400000","o":"EA380B","s":"_ZN5mongo15invariantFailedEPKcS1_j"},{"b":"400000","o":"CE766D"},{"b":"400000","o":"CC6643"},{"b":"400000","o":"CC763C"},{"b":"400000","o":"CC7BA7"},{"b":"400000","o":"CC82D9"},{"b":"400000","o":"F35CB1","s":"_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code"},{"b":"400000","o":"F35ED1","s":"_ZN4asio6detail9scheduler3runERSt10error_code"},{"b":"400000","o":"F3A06F","s":"_ZN4asio10io_service3runEv"},{"b":"400000","o":"CD36B5"},{"b":"400000","o":"1734160","s":"execute_native_thread_routine"},{"b":"7F9128FE6000","o":"8184"},{"b":"7F9128C21000","o":"FA37D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.2.7", "gitVersion" : "4249c1d2b5999ebbf1fdf3bc0e0e3b3ff5c0aaf2", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.13.0-86-generic", "version" : "#131-Ubuntu SMP Thu May 12 23:33:13 UTC 2016", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "A8DC02B241532EEAD31FFE2633EE8C8E565C1F1A" }, { "b" : "7FFEDAEF3000", "elfType" : 3, "buildId" : "FB51781B01C799D4B6F6FED07F9862B3B24B375F" }, { "b" : "7F9129F08000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "74864DB9D5F69D39A67E4755012FB6573C469B3D" }, { "b" : "7F9129B2C000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "AAE7CFF8351B730830BDBCE0DCABBE06574B7144" }, { "b" : "7F9129924000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "E2A6DD5048A0A051FD61043BDB69D8CC68192AB7" }, { "b" : "7F9129720000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "DA9B8C234D0FE9FD8CAAC8970A7EC1B6C8F6623F" }, { "b" : "7F912941A000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "D144258E614900B255A31F3FD2283A878670D5BC" }, { "b" : "7F9129204000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "36311B4457710AE5578C4BF00791DED7359DBB92" }, { "b" : "7F9128FE6000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "31E9F21AE8C10396171F1E13DA15780986FA696C" }, { "b" : "7F9128C21000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "CF699A15CAAE64F50311FC4655B86DC39A479789" }, { "b" : "7F912A167000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "D0F537904076D73F29E4A37341F8A449E2EF6CD0" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x1319fa2]
 mongod(+0xF190F9) [0x13190f9]
 mongod(+0xF19902) [0x1319902]
 libpthread.so.0(+0x10330) [0x7f9128ff6330]
 libc.so.6(gsignal+0x37) [0x7f9128c57c37]
 libc.so.6(abort+0x148) [0x7f9128c5b028]
 mongod(_ZN5mongo15invariantFailedEPKcS1_j+0xCB) [0x12a380b]
 mongod(+0xCE766D) [0x10e766d]
 mongod(+0xCC6643) [0x10c6643]
 mongod(+0xCC763C) [0x10c763c]
 mongod(+0xCC7BA7) [0x10c7ba7]
 mongod(+0xCC82D9) [0x10c82d9]
 mongod(_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code+0x2F1) [0x1335cb1]
 mongod(_ZN4asio6detail9scheduler3runERSt10error_code+0xC1) [0x1335ed1]
 mongod(_ZN4asio10io_service3runEv+0x2F) [0x133a06f]
 mongod(+0xCD36B5) [0x10d36b5]
 mongod(execute_native_thread_routine+0x20) [0x1b34160]
 libpthread.so.0(+0x8184) [0x7f9128fee184]
 libc.so.6(clone+0x6D) [0x7f9128d1b37d]
-----  END BACKTRACE  -----

Comment by Githook User [ 30/Jun/16 ]

Author:

{u'username': u'Machyne', u'name': u'Matt Cotter', u'email': u'matt.cotter@mongodb.com'}

Message: SERVER-24711 properly timeout asio connections

ASIO Connections that have already timed out before setup now result
in timeouts instead of invariants.

(cherry picked from commit 587afac9ca7ff084aeb8bf74e98ecb9688b3afdb)
Branch: v3.2
https://github.com/mongodb/mongo/commit/67602c79678d91e62b0c1bf125743ff992683a16

Comment by Ramon Fernandez Marina [ 29/Jun/16 ]

Sorry you've run into this again edikmkoyan – we have a fix in the master branch (released yesterday as part of 3.3.9) that is being tested for backport to 3.2. Please continue to watch the ticket for updates.

Comment by Edik Mkoyan [ 29/Jun/16 ]

It happened when master lost connection to arbiter and slave

2016-06-29T10:55:59.101+0400 I REPL     [ReplicationExecutor] Error in heartbeat request to evn-mrs-slave:27017; ExceededTimeLimit: Couldn't get a connection within the time limit
2016-06-29T10:55:59.871+0400 I NETWORK  [initandlisten] connection accepted from 10.24.20.163:40393 #326 (4 connections now open)
2016-06-29T10:55:59.900+0400 I REPL     [ReplicationExecutor] Error in heartbeat request to evn-mrs-arbiter:27017; ExceededTimeLimit: Operation timed out
2016-06-29T10:55:59.904+0400 I ACCESS   [conn326] Successfully authenticated as principal __system on local
2016-06-29T10:56:00.951+0400 I NETWORK  [initandlisten] connection accepted from 10.24.20.164:57887 #327 (5 connections now open)
2016-06-29T10:56:00.986+0400 I ACCESS   [conn327] Successfully authenticated as principal __system on local
2016-06-29T10:56:07.924+0400 I NETWORK  [ReplicaSetMonitorWatcher] getaddrinfo("mcfg03") failed: Temporary failure in name resolution
2016-06-29T10:56:07.924+0400 I NETWORK  [replSetDistLockPinger] getaddrinfo("mcfg01") failed: Temporary failure in name resolution
2016-06-29T10:56:07.924+0400 W NETWORK  [replSetDistLockPinger] No primary detected for set mcfg
2016-06-29T10:56:07.924+0400 W SHARDING [replSetDistLockPinger] pinging failed for distributed lock pinger :: caused by :: FailedToSatisfyReadPreference: could not find host matching read preference { mode: "primary" } for set mcfg
2016-06-29T10:56:07.925+0400 I NETWORK  [initandlisten] connection accepted from 10.44.20.190:42830 #328 (6 connections now open)
2016-06-29T10:56:07.946+0400 I ACCESS   [conn328] Successfully authenticated as principal __system on local
2016-06-29T10:56:07.983+0400 I NETWORK  [initandlisten] connection accepted from 10.44.20.165:53254 #329 (7 connections now open)
2016-06-29T10:56:08.007+0400 I ACCESS   [conn329] Successfully authenticated as principal __system on local
2016-06-29T10:56:08.437+0400 I REPL     [ReplicationExecutor] Member evn-mrs-arbiter:27017 is now in state ARBITER
2016-06-29T10:56:08.438+0400 I REPL     [ReplicationExecutor] Standing for election
2016-06-29T10:56:08.438+0400 I REPL     [ReplicationExecutor] not electing self, we are not freshest
2016-06-29T10:56:08.438+0400 I REPL     [ReplicationExecutor] not electing self, we are not freshest
2016-06-29T10:56:08.994+0400 I NETWORK  [initandlisten] connection accepted from 10.44.20.191:52197 #330 (8 connections now open)
2016-06-29T10:56:09.033+0400 I ACCESS   [conn330] Successfully authenticated as principal __system on local
2016-06-29T10:56:09.101+0400 I -        [NetworkInterfaceASIO-Replication-0] Invariant failure _connection.is_initialized() src/mongo/executor/network_interface_asio_operation.cpp 142
2016-06-29T10:56:09.101+0400 I -        [NetworkInterfaceASIO-Replication-0] 
 
***aborting after invariant() failure
 
 
2016-06-29T10:56:09.116+0400 F -        [NetworkInterfaceASIO-Replication-0] Got signal: 6 (Aborted).
 
 0x1319fa2 0x13190f9 0x1319902 0x7f34070fb330 0x7f3406d5cc37 0x7f3406d60028 0x12a380b 0x10e766d 0x10c6643 0x10c763c 0x10c7ba7 0x10c82d9 0x1335cb1 0x1335ed1 0x133a06f 0x10d36b5 0x1b34160 0x7f34070f3184 0x7f3406e2037d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"F19FA2","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"F190F9"},{"b":"400000","o":"F19902"},{"b":"7F34070EB000","o":"10330"},{"b":"7F3406D26000","o":"36C37","s":"gsignal"},{"b":"7F3406D26000","o":"3A028","s":"abort"},{"b":"400000","o":"EA380B","s":"_ZN5mongo15invariantFailedEPKcS1_j"},{"b":"400000","o":"CE766D"},{"b":"400000","o":"CC6643"},{"b":"400000","o":"CC763C"},{"b":"400000","o":"CC7BA7"},{"b":"400000","o":"CC82D9"},{"b":"400000","o":"F35CB1","s":"_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code"},{"b":"400000","o":"F35ED1","s":"_ZN4asio6detail9scheduler3runERSt10error_code"},{"b":"400000","o":"F3A06F","s":"_ZN4asio10io_service3runEv"},{"b":"400000","o":"CD36B5"},{"b":"400000","o":"1734160","s":"execute_native_thread_routine"},{"b":"7F34070EB000","o":"8184"},{"b":"7F3406D26000","o":"FA37D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.2.7", "gitVersion" : "4249c1d2b5999ebbf1fdf3bc0e0e3b3ff5c0aaf2", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.19.0-49-generic", "version" : "#55~14.04.1-Ubuntu SMP Fri Jan 22 11:24:31 UTC 2016", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "A8DC02B241532EEAD31FFE2633EE8C8E565C1F1A" }, { "b" : "7FFC3A9A6000", "elfType" : 3, "buildId" : "BD3BE3710E37C8678F035B516346C5E37B10C936" }, { "b" : "7F340800D000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "74864DB9D5F69D39A67E4755012FB6573C469B3D" }, { "b" : "7F3407C31000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "AAE7CFF8351B730830BDBCE0DCABBE06574B7144" }, { "b" : "7F3407A29000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "E2A6DD5048A0A051FD61043BDB69D8CC68192AB7" }, { "b" : "7F3407825000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "DA9B8C234D0FE9FD8CAAC8970A7EC1B6C8F6623F" }, { "b" : "7F340751F000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "D144258E614900B255A31F3FD2283A878670D5BC" }, { "b" : "7F3407309000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "36311B4457710AE5578C4BF00791DED7359DBB92" }, { "b" : "7F34070EB000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "31E9F21AE8C10396171F1E13DA15780986FA696C" }, { "b" : "7F3406D26000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "CF699A15CAAE64F50311FC4655B86DC39A479789" }, { "b" : "7F340826C000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "D0F537904076D73F29E4A37341F8A449E2EF6CD0" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x1319fa2]
 mongod(+0xF190F9) [0x13190f9]
 mongod(+0xF19902) [0x1319902]
 libpthread.so.0(+0x10330) [0x7f34070fb330]
 libc.so.6(gsignal+0x37) [0x7f3406d5cc37]
 libc.so.6(abort+0x148) [0x7f3406d60028]
 mongod(_ZN5mongo15invariantFailedEPKcS1_j+0xCB) [0x12a380b]
 mongod(+0xCE766D) [0x10e766d]
 mongod(+0xCC6643) [0x10c6643]
 mongod(+0xCC763C) [0x10c763c]
 mongod(+0xCC7BA7) [0x10c7ba7]
 mongod(+0xCC82D9) [0x10c82d9]
 mongod(_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code+0x2F1) [0x1335cb1]
 mongod(_ZN4asio6detail9scheduler3runERSt10error_code+0xC1) [0x1335ed1]
 mongod(_ZN4asio10io_service3runEv+0x2F) [0x133a06f]
 mongod(+0xCD36B5) [0x10d36b5]
 mongod(execute_native_thread_routine+0x20) [0x1b34160]
 libpthread.so.0(+0x8184) [0x7f34070f3184]
 libc.so.6(clone+0x6D) [0x7f3406e2037d]
-----  END BACKTRACE  -----

Comment by Githook User [ 24/Jun/16 ]

Author:

{u'username': u'Machyne', u'name': u'Matt Cotter', u'email': u'matt.cotter@mongodb.com'}

Message: SERVER-24711 properly timeout asio connections

ASIO Connections that have already timed out before setup now result
in timeouts instead of invariants.
Branch: master
https://github.com/mongodb/mongo/commit/587afac9ca7ff084aeb8bf74e98ecb9688b3afdb

Comment by Aristarkh Zagorodnikov [ 23/Jun/16 ]

Same stuff here. 3.2.7, both mongos and mongod crash when one of RS nodes goes away:

mongos

2016-06-23T11:27:30.445+0000 I -        [NetworkInterfaceASIO-ShardRegistry-0] Invariant failure _connection.is_initialized() src/mo
ngo/executor/network_interface_asio_operation.cpp 142
2016-06-23T11:27:30.445+0000 I -        [NetworkInterfaceASIO-ShardRegistry-0]
 
***aborting after invariant() failure
 
 
2016-06-23T11:27:30.459+0000 F -        [NetworkInterfaceASIO-ShardRegistry-0] Got signal: 6 (Aborted).
 
 0xc62cf2 0xc61e49 0xc62652 0x7f21e0cbb330 0x7f21e091cc37 0x7f21e0920028 0xbeadab 0xa22b0d 0xa018c3 0xa028bc 0xa02e27 0xa03559 0xc7e
981 0xc7eba1 0xc82d3f 0xa0ea25 0xe958e0 0x7f21e0cb3184 0x7f21e09e037d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"862CF2","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"861E49"},{"b":"400000","o":"862652
"},{"b":"7F21E0CAB000","o":"10330"},{"b":"7F21E08E6000","o":"36C37","s":"gsignal"},{"b":"7F21E08E6000","o":"3A028","s":"abort"},{"b"
:"400000","o":"7EADAB","s":"_ZN5mongo15invariantFailedEPKcS1_j"},{"b":"400000","o":"622B0D"},{"b":"400000","o":"6018C3"},{"b":"40000
0","o":"6028BC"},{"b":"400000","o":"602E27"},{"b":"400000","o":"603559"},{"b":"400000","o":"87E981","s":"_ZN4asio6detail9scheduler10
do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code"},{"b":"400000","o":"87EBA1","s":"_Z
N4asio6detail9scheduler3runERSt10error_code"},{"b":"400000","o":"882D3F","s":"_ZN4asio10io_service3runEv"},{"b":"400000","o":"60EA25
"},{"b":"400000","o":"A958E0","s":"execute_native_thread_routine"},{"b":"7F21E0CAB000","o":"8184"},{"b":"7F21E08E6000","o":"FA37D","
s":"clone"}],"processInfo":{ "mongodbVersion" : "3.2.7", "gitVersion" : "4249c1d2b5999ebbf1fdf3bc0e0e3b3ff5c0aaf2", "compiledModules
" : [], "uname" : { "sysname" : "Linux", "release" : "4.4.13-moby", "version" : "#1 SMP Fri Jun 17 22:06:24 UTC 2016", "machine" : "
x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "F1B509A387A65E9AB9E969F4905CC875D7FFA652" }, { "b" : "7FFF83342
000", "elfType" : 3, "buildId" : "E470B8A5CB8F11BBB914DD362A5F29B92312D55D" }, { "b" : "7F21E1BCD000", "path" : "/lib/x86_64-linux-g
nu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "74864DB9D5F69D39A67E4755012FB6573C469B3D" }, { "b" : "7F21E17F1000", "path" : "/lib
/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "AAE7CFF8351B730830BDBCE0DCABBE06574B7144" }, { "b" : "7F21E15E900
0", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "E2A6DD5048A0A051FD61043BDB69D8CC68192AB7" }, { "b" : "7
F21E13E5000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "DA9B8C234D0FE9FD8CAAC8970A7EC1B6C8F6623F" },
{ "b" : "7F21E10DF000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "D144258E614900B255A31F3FD2283A878670
D5BC" }, { "b" : "7F21E0EC9000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "36311B4457710AE5578C4BF
00791DED7359DBB92" }, { "b" : "7F21E0CAB000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "31E9F21A
E8C10396171F1E13DA15780986FA696C" }, { "b" : "7F21E08E6000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" :
"CF699A15CAAE64F50311FC4655B86DC39A479789" }, { "b" : "7F21E1E2C000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildI
d" : "D0F537904076D73F29E4A37341F8A449E2EF6CD0" } ] }}
 mongos(_ZN5mongo15printStackTraceERSo+0x32) [0xc62cf2]
 mongos(+0x861E49) [0xc61e49]
 mongos(+0x862652) [0xc62652]
 libpthread.so.0(+0x10330) [0x7f21e0cbb330]
 libc.so.6(gsignal+0x37) [0x7f21e091cc37]
 libc.so.6(abort+0x148) [0x7f21e0920028]
 mongos(_ZN5mongo15invariantFailedEPKcS1_j+0xCB) [0xbeadab]
 mongos(+0x622B0D) [0xa22b0d]
 mongos(+0x6018C3) [0xa018c3]
 mongos(+0x6028BC) [0xa028bc]
 mongos(+0x602E27) [0xa02e27]
 mongos(+0x603559) [0xa03559]
 mongos(_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code+0
x2F1) [0xc7e981]
 mongos(_ZN4asio6detail9scheduler3runERSt10error_code+0xC1) [0xc7eba1]
 mongos(_ZN4asio10io_service3runEv+0x2F) [0xc82d3f]
 mongos(+0x60EA25) [0xa0ea25]
 mongos(execute_native_thread_routine+0x20) [0xe958e0]
 libpthread.so.0(+0x8184) [0x7f21e0cb3184]
 libc.so.6(clone+0x6D) [0x7f21e09e037d]
-----  END BACKTRACE  -----

mongod

2016-06-23T11:27:07.349+0000 I -        [NetworkInterfaceASIO-Replication-0] Invariant failure _connection.is_initialized() src/mong
o/executor/network_interface_asio_operation.cpp 142
2016-06-23T11:27:07.349+0000 I -        [NetworkInterfaceASIO-Replication-0]
 
***aborting after invariant() failure
 
 
2016-06-23T11:27:07.358+0000 F -        [NetworkInterfaceASIO-Replication-0] Got signal: 6 (Aborted).
 
 0x1319fa2 0x13190f9 0x1319902 0x7f009ee28330 0x7f009ea89c37 0x7f009ea8d028 0x12a380b 0x10e766d 0x10c6643 0x10c763c 0x10c7ba7 0x10c8
2d9 0x1335cb1 0x1335ed1 0x133a06f 0x10d36b5 0x1b34160 0x7f009ee20184 0x7f009eb4d37d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"F19FA2","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"F190F9"},{"b":"400000","o":"F19902
"},{"b":"7F009EE18000","o":"10330"},{"b":"7F009EA53000","o":"36C37","s":"gsignal"},{"b":"7F009EA53000","o":"3A028","s":"abort"},{"b"
:"400000","o":"EA380B","s":"_ZN5mongo15invariantFailedEPKcS1_j"},{"b":"400000","o":"CE766D"},{"b":"400000","o":"CC6643"},{"b":"40000
0","o":"CC763C"},{"b":"400000","o":"CC7BA7"},{"b":"400000","o":"CC82D9"},{"b":"400000","o":"F35CB1","s":"_ZN4asio6detail9scheduler10
do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code"},{"b":"400000","o":"F35ED1","s":"_Z
N4asio6detail9scheduler3runERSt10error_code"},{"b":"400000","o":"F3A06F","s":"_ZN4asio10io_service3runEv"},{"b":"400000","o":"CD36B5
"},{"b":"400000","o":"1734160","s":"execute_native_thread_routine"},{"b":"7F009EE18000","o":"8184"},{"b":"7F009EA53000","o":"FA37D",
"s":"clone"}],"processInfo":{ "mongodbVersion" : "3.2.7", "gitVersion" : "4249c1d2b5999ebbf1fdf3bc0e0e3b3ff5c0aaf2", "compiledModule
s" : [], "uname" : { "sysname" : "Linux", "release" : "4.4.13-moby", "version" : "#1 SMP Fri Jun 17 22:06:24 UTC 2016", "machine" :
"x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "A8DC02B241532EEAD31FFE2633EE8C8E565C1F1A" }, { "b" : "7FFD7EEA
4000", "elfType" : 3, "buildId" : "E470B8A5CB8F11BBB914DD362A5F29B92312D55D" }, { "b" : "7F009FD3A000", "path" : "/lib/x86_64-linux-
gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "74864DB9D5F69D39A67E4755012FB6573C469B3D" }, { "b" : "7F009F95E000", "path" : "/li
b/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "AAE7CFF8351B730830BDBCE0DCABBE06574B7144" }, { "b" : "7F009F7560
00", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "E2A6DD5048A0A051FD61043BDB69D8CC68192AB7" }, { "b" : "
7F009F552000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "DA9B8C234D0FE9FD8CAAC8970A7EC1B6C8F6623F" },
 { "b" : "7F009F24C000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "D144258E614900B255A31F3FD2283A87867
0D5BC" }, { "b" : "7F009F036000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "36311B4457710AE5578C4B
F00791DED7359DBB92" }, { "b" : "7F009EE18000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "31E9F21
AE8C10396171F1E13DA15780986FA696C" }, { "b" : "7F009EA53000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" :
 "CF699A15CAAE64F50311FC4655B86DC39A479789" }, { "b" : "7F009FF99000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "build
Id" : "D0F537904076D73F29E4A37341F8A449E2EF6CD0" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x1319fa2]
 mongod(+0xF190F9) [0x13190f9]
 mongod(+0xF19902) [0x1319902]
 libpthread.so.0(+0x10330) [0x7f009ee28330]
 libc.so.6(gsignal+0x37) [0x7f009ea89c37]
 libc.so.6(abort+0x148) [0x7f009ea8d028]
 mongod(_ZN5mongo15invariantFailedEPKcS1_j+0xCB) [0x12a380b]
 mongod(+0xCE766D) [0x10e766d]
 mongod(+0xCC6643) [0x10c6643]
 mongod(+0xCC763C) [0x10c763c]
 mongod(+0xCC7BA7) [0x10c7ba7]
 mongod(+0xCC82D9) [0x10c82d9]
 mongod(_ZN4asio6detail9scheduler10do_run_oneERNS0_11scoped_lockINS0_11posix_mutexEEERNS0_21scheduler_thread_infoERKSt10error_code+0
x2F1) [0x1335cb1]
 mongod(_ZN4asio6detail9scheduler3runERSt10error_code+0xC1) [0x1335ed1]
 mongod(_ZN4asio10io_service3runEv+0x2F) [0x133a06f]
 mongod(+0xCD36B5) [0x10d36b5]
 mongod(execute_native_thread_routine+0x20) [0x1b34160]
 libpthread.so.0(+0x8184) [0x7f009ee20184]
 libc.so.6(clone+0x6D) [0x7f009eb4d37d]
-----  END BACKTRACE  -----

Comment by Ramon Fernandez Marina [ 22/Jun/16 ]

The buildId is from this binary, and the debug symbols are here. Below is the output of addr2line:

/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/mongo/util/stacktrace_posix.cpp:172
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/mongo/util/signal_handlers_synchronous.cpp:182
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/mongo/util/signal_handlers_synchronous.cpp:238
??:0
??:0
??:0
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/mongo/util/assert_util.cpp:154 (discriminator 1)
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/mongo/executor/network_interface_asio_operation.cpp:142 (discriminator 1)
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/mongo/executor/connection_pool_asio.cpp:192
/opt/mongodbtoolchain/v1/include/c++/4.8.2/functional:2071
/opt/mongodbtoolchain/v1/include/c++/4.8.2/functional:2029
/opt/mongodbtoolchain/v1/include/c++/4.8.2/functional:2174
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/mongo/executor/connection_pool_asio.cpp:87
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/bind_handler.hpp:62
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/handler_invoke_hook.hpp:68
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/handler_invoke_helpers.hpp:37
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/bind_handler.hpp:103
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/handler_invoke_helpers.hpp:37
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/impl/strand_service.hpp:61
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/gcc_x86_fenced_block.hpp:82
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/gcc_x86_fenced_block.hpp:50
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/completion_handler.hpp:70
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/impl/strand_service.hpp:83
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/io_service_strand.hpp:207
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/wrapped_handler.hpp:231
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/handler_invoke_helpers.hpp:37
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/handler_work.hpp:81
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/wait_handler.hpp:71
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/impl/scheduler.ipp:60
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/impl/scheduler.ipp:371
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/impl/scheduler.ipp:146 (discriminator 1)
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/throw_error.hpp:33
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/impl/io_service.ipp:61
/opt/mongodbtoolchain/v1/include/c++/4.8.2/bits/atomic_base.h:412
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/detail/scheduler.hpp:85
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/third_party/asio-asio-1-11-0/asio/include/asio/impl/io_service.hpp:264
/data/mci/e3d7907c7f9e7fd05cc3e66f3c8a7a83/src/src/mongo/executor/network_interface_asio.cpp:116
/opt/mongodbtoolchain/v1/include/c++/4.8.2/functional:1732
/opt/mongodbtoolchain/v1/include/c++/4.8.2/functional:1720
/opt/mongodbtoolchain/v1/include/c++/4.8.2/thread:115
/data/mci/8b24fd5eb2023a18a85d7c8d7c44dac1/toolchain-builder/build-gcc-v1.sh-Bi5/x86_64-mongodb-linux/libstdc++-v3/include/bits/shared_ptr_base.h:545
/data/mci/8b24fd5eb2023a18a85d7c8d7c44dac1/toolchain-builder/build-gcc-v1.sh-Bi5/x86_64-mongodb-linux/libstdc++-v3/include/bits/shared_ptr_base.h:781
/data/mci/8b24fd5eb2023a18a85d7c8d7c44dac1/toolchain-builder/build-gcc-v1.sh-Bi5/x86_64-mongodb-linux/libstdc++-v3/include/bits/shared_ptr.h:93
/data/mci/8b24fd5eb2023a18a85d7c8d7c44dac1/toolchain-builder/build-gcc-v1.sh-Bi5/x86_64-mongodb-linux/libstdc++-v3/src/c++11/../../../../../gcc-4.8.2/libstdc++-v3/src/c++11/thread.cc:95
??:0
??:0

Generated at Thu Feb 08 04:07:11 UTC 2024 using Jira 9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66.