<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:12:46 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-26654] ExceededTimeLimit: Operation timed out after uprgade from 3.2.8 to 3.2.10</title>
                <link>https://jira.mongodb.org/browse/SERVER-26654</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;We are running our production cluster using version 3.2.8, Our cluster consists of four mongos , three config servers and ten shards-3-node-replica-sets , we perform both primary and secondary reads. Very frequently we hit &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-26159&quot; title=&quot;mongos crashes:  Invariant failure !op-&amp;gt;timedOut() src/mongo/executor/network_interface_asio.cpp&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-26159&quot;&gt;&lt;del&gt;SERVER-26159&lt;/del&gt;&lt;/a&gt; and our mongos crashes. According to the bug report the issue was solved on 3.2.10 and we upgraded on Oct 11. 10 minutes after the upgrade completed we start receiving errors that the mongos can&apos;t connect to the shards due to timeout. Timeouts were very frequent and reduce our processing throughput to 20%. We forced to rollback to 3.2.8 and the issue gone away, we don&apos;t receive any timeout but the &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-26159&quot; title=&quot;mongos crashes:  Invariant failure !op-&amp;gt;timedOut() src/mongo/executor/network_interface_asio.cpp&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-26159&quot;&gt;&lt;del&gt;SERVER-26159&lt;/del&gt;&lt;/a&gt; returned. &lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Oct 11 06:40:03 [NetworkInterfaceASIO-TaskExecutorPool-1-0] Successfully connected to node1-48668:48668&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Oct 11 06:40:03 mongos16026 mongos.16026[27341]: [NetworkInterfaceASIO-TaskExecutorPool-1-0] Successfully connected to node0-49179:49179&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Oct 11 06:40:03 mongos16026 mongos.16026[27341]: [NetworkInterfaceASIO-TaskExecutorPool-4-0] Connecting to node0-49627:49627&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Oct 11 06:40:03 mongos16026 mongos.16026[27341]: [NetworkInterfaceASIO-TaskExecutorPool-1-0] Failed to connect to node0-49630:49630 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Oct 11 06:40:03 mongos16026 mongos.16026[27341]: [NetworkInterfaceASIO-TaskExecutorPool-4-0] Connecting to node1-49631:49631&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Oct 11 06:40:03 mongos16026 mongos.16026[27341]: [NetworkInterfaceASIO-TaskExecutorPool-4-0] Connecting to node0-48668:48668&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Oct 11 06:40:03 mongos16026 mongos.16026[27341]: [NetworkInterfaceASIO-TaskExecutorPool-4-0] Connecting to node0-48574:48574&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Oct 11 06:40:03 mongos16026 mongos.16026[27341]: [NetworkInterfaceASIO-TaskExecutorPool-1-0] Failed to connect to node0-49631:49631 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Oct 11 06:40:03 mongos16026 mongos.16026[27341]: [NetworkInterfaceASIO-TaskExecutorPool-1-0] Failed to connect to node0-49631:49631 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Oct 11 06:40:03 mongos16026 mongos.16026[27341]: [NetworkInterfaceASIO-TaskExecutorPool-1-0] Failed to connect to node1-48574:48574 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Is this a known bug in 3.2.10 and is there any workaround?&lt;/p&gt;</description>
                <environment></environment>
        <key id="324096">SERVER-26654</key>
            <summary>ExceededTimeLimit: Operation timed out after uprgade from 3.2.8 to 3.2.10</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="3">Duplicate</resolution>
                                        <assignee username="kaloian.manassiev@mongodb.com">Kaloian Manassiev</assignee>
                                    <reporter username="antogiann">Antonis Giannopoulos</reporter>
                        <labels>
                    </labels>
                <created>Mon, 17 Oct 2016 08:08:23 +0000</created>
                <updated>Tue, 11 Apr 2017 15:57:31 +0000</updated>
                            <resolved>Fri, 11 Nov 2016 16:54:46 +0000</resolved>
                                    <version>3.2.10</version>
                                                    <component>Networking</component>
                    <component>Sharding</component>
                                        <votes>2</votes>
                                    <watches>18</watches>
                                                                                                                <comments>
                            <comment id="1432661" author="alessandro.gherardi@yahoo.com" created="Sat, 12 Nov 2016 20:05:23 +0000"  >&lt;p&gt;Here&apos;s the log files, broken down per server.&lt;/p&gt;

&lt;p&gt;The environment consists of 3 Windows 2012 servers. There is a single replicaset with 3 mongoD&apos;s - one on each server - and CSRS with 3 config servers - one on each server . There is a mongoS on each server . The IP&apos;s are 192.168.20.201 (mongod0-rs1-GUID and configsvr1-GUID), 192.168.20.202 (mongod2-rs1-GUID and configsvr2-GUID), and 192.168.20.203 (mongod1-rs1-GUID and configsvr3-GUID).&lt;/p&gt;

&lt;p&gt;Notice that I rebooted the servers prior to running the test - this is a test environment - so connection failures earlier in the logs are expected.&lt;/p&gt;</comment>
                            <comment id="1432077" author="kaloian.manassiev" created="Fri, 11 Nov 2016 16:54:47 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=antogiann&quot; class=&quot;user-hover&quot; rel=&quot;antogiann&quot;&gt;antogiann&lt;/a&gt; I am closing this ticket and moving the investigation to happen in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-26723&quot; title=&quot;Mongos stalls for even simple queries&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-26723&quot;&gt;&lt;del&gt;SERVER-26723&lt;/del&gt;&lt;/a&gt;.&lt;/p&gt;</comment>
                            <comment id="1431328" author="kaloian.manassiev" created="Thu, 10 Nov 2016 18:47:27 +0000"  >&lt;p&gt;Yes, I meant the mongod node to which mongos failed to connect. For example, this would be node &lt;tt&gt;node0-49630:49630&lt;/tt&gt; if this was the error message:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Failed to connect to node0-49630:49630 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;-Kal.&lt;/p&gt;</comment>
                            <comment id="1431271" author="alessandro.gherardi@yahoo.com" created="Thu, 10 Nov 2016 17:53:25 +0000"  >&lt;p&gt;Kal,&lt;/p&gt;

&lt;p&gt;&amp;gt;&amp;gt;&amp;gt;&amp;gt; the log from the host against which the call was made?&lt;br/&gt;
You mean the mongoD log?&lt;/p&gt;

&lt;p&gt;&amp;gt;&amp;gt;&amp;gt;&amp;gt; would it be possible that you try to use 3.2.11-rc0 again &lt;br/&gt;
This weekend, I should have access to the test environment where we noticed this issue. I&apos;ll retry 3.2.11-rc0 and attach the info you requested to this ticket&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Alessandro&lt;/p&gt;</comment>
                            <comment id="1431250" author="kaloian.manassiev" created="Thu, 10 Nov 2016 17:31:42 +0000"  >&lt;p&gt;Thank you &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=antogiann&quot; class=&quot;user-hover&quot; rel=&quot;antogiann&quot;&gt;antogiann&lt;/a&gt; and &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=agherardi&quot; class=&quot;user-hover&quot; rel=&quot;agherardi&quot;&gt;agherardi&lt;/a&gt; for trying out 3.2.11-rc0. It is unfortunate that our hypothesis was not correct and also we are unable to reproduce it while running sysbench in AWS like &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=antogiann&quot; class=&quot;user-hover&quot; rel=&quot;antogiann&quot;&gt;antogiann&lt;/a&gt; suggested.&lt;/p&gt;

&lt;p&gt;However, in 3.2.11-rc0 we improved the logging around &lt;a href=&quot;https://github.com/mongodb/mongo/commit/ce95ca7f27b44e834baf2bd7d0dff53c8d09ea56&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;network timeouts&lt;/a&gt;, which should be useful in tracking this down.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=agherardi&quot; class=&quot;user-hover&quot; rel=&quot;agherardi&quot;&gt;agherardi&lt;/a&gt;, would it be possible that you try to use 3.2.11-rc0 again and increase the log verbosity for the command, network and asio components using the calls below, until the issue reproduces and then include the mongos log and the log from the host against which the call was made?&lt;/p&gt;

&lt;p&gt;This is the command to increase log verbosity:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;db.adminCommand({ setParameter: 1, logComponentVerbosity: { command: { verbosity: 1 } } });&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;db.adminCommand({ setParameter: 1, logComponentVerbosity: { network: 1 } });&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;db.adminCommand({ setParameter: 1, logComponentVerbosity: { network: { asio: { verbosity: 2 } } } });&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;And this is the command to bring it back to zero:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;db.adminCommand({ setParameter: 1, logComponentVerbosity: { command: { verbosity: 0 } } });&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;db.adminCommand({ setParameter: 1, logComponentVerbosity: { network: 0 } });&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;db.adminCommand({ setParameter: 1, logComponentVerbosity: { network: { asio: { verbosity: 0 } } } });&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Thank you very much in advance for your help.&lt;/p&gt;

&lt;p&gt;-Kal.&lt;/p&gt;</comment>
                            <comment id="1431119" author="antogiann" created="Thu, 10 Nov 2016 15:46:17 +0000"  >&lt;p&gt;Hi Kal,&lt;/p&gt;

&lt;p&gt;I tested using the same methodology against 3.2.11-rc0 few minutes ago and the issue is still there. &lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Antonis&lt;/p&gt;</comment>
                            <comment id="1431094" author="royrez@microsoft.com" created="Thu, 10 Nov 2016 15:32:52 +0000"  >&lt;p&gt;We have the same issue with 3.2.10.&lt;/p&gt;</comment>
                            <comment id="1430540" author="alessandro.gherardi@yahoo.com" created="Wed, 9 Nov 2016 20:48:28 +0000"  >&lt;p&gt;We have the same problem. We just tried 3.2.11-rc0, same issue.&lt;/p&gt;</comment>
                            <comment id="1430287" author="antogiann" created="Wed, 9 Nov 2016 16:38:33 +0000"  >&lt;p&gt;Thanks Kal,&lt;/p&gt;

&lt;p&gt;I will try mongo-sysbench configuration that reproduced the issue in 3.2.10 against 3.2.11-rc0 (when its out) and share the results with you. &lt;/p&gt;

&lt;p&gt;I will check the repo daily but It would be great if you give me a heads up when its out.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Antonis&lt;/p&gt;</comment>
                            <comment id="1430286" author="jonhyman" created="Wed, 9 Nov 2016 16:37:50 +0000"  >&lt;p&gt;Okay. &lt;/p&gt;</comment>
                            <comment id="1430159" author="kaloian.manassiev" created="Wed, 9 Nov 2016 14:55:59 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=jon%40appboy.com&quot; class=&quot;user-hover&quot; rel=&quot;jon@appboy.com&quot;&gt;jon@appboy.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Yes - based on the symptoms described in this ticket we expect that &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-26859&quot; title=&quot;AsyncResultsMerger replica set retargeting may block the ASIO callback threads&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-26859&quot;&gt;&lt;del&gt;SERVER-26859&lt;/del&gt;&lt;/a&gt; should fix them.&lt;/p&gt;

&lt;p&gt;The 3.2.11-rc0 release candidate is expected to become available in the coming days. In case you are hitting this problem, would it be possible to give it a try and let us know whether it solves it for you?&lt;/p&gt;

&lt;p&gt;Thanks in advance.&lt;/p&gt;

&lt;p&gt;Best regards,&lt;br/&gt;
-Kal.&lt;/p&gt;</comment>
                            <comment id="1430110" author="jonhyman" created="Wed, 9 Nov 2016 13:30:11 +0000"  >&lt;p&gt;Is this fixed now that &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-26859&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;https://jira.mongodb.org/browse/SERVER-26859&lt;/a&gt; is done?&lt;/p&gt;</comment>
                            <comment id="1423030" author="kaloian.manassiev" created="Tue, 1 Nov 2016 15:57:04 +0000"  >&lt;p&gt;Thanks &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=antogiann&quot; class=&quot;user-hover&quot; rel=&quot;antogiann&quot;&gt;antogiann&lt;/a&gt; for the information. Unfortunately these filtered-out logs do not give us enough information about what is causing the connections to time-out, but I am now working on reproducing the problem using your sysbench configuration.&lt;/p&gt;

&lt;p&gt;Also, from a different report with similar symptoms we now have a sense of what might be going on. The problem is described in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-26859&quot; title=&quot;AsyncResultsMerger replica set retargeting may block the ASIO callback threads&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-26859&quot;&gt;&lt;del&gt;SERVER-26859&lt;/del&gt;&lt;/a&gt; and we are now working on getting it fixed and ported to 3.2.&lt;/p&gt;

&lt;p&gt;Please watch this ticket for further updates.&lt;/p&gt;</comment>
                            <comment id="1416693" author="antogiann" created="Tue, 25 Oct 2016 13:14:49 +0000"  >&lt;p&gt;Hi Kal,&lt;/p&gt;

&lt;p&gt;I manage to replicate the error using mongo-sysbench. My setup was six &quot;sbtest&quot; collections with 40 million documents each sharded on &lt;/p&gt;
{ &quot;_id&quot; : &quot;hashed&quot; }
&lt;p&gt;. The sysbench config options may found attached. I used 9 shards of MMAPv1 3.2.10 and config servers on SCCC topology. All components (shards,config,mongos) were running CentOS 6.&lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
Antonis&lt;/p&gt;</comment>
                            <comment id="1411283" author="antogiann" created="Tue, 18 Oct 2016 12:48:35 +0000"  >&lt;p&gt;Hi Kal,&lt;/p&gt;

&lt;p&gt;Thank you for looking at this. &lt;/p&gt;

&lt;p&gt;The logs are from one of my mongos but all four reported the same behavior. Our audit team is examine the logs and I am confident that I will be able to share the requested information with you.  &lt;/p&gt;

&lt;p&gt;Are you running your config servers in SCCC mode or in CSRS? &amp;#8211; SCCC , it was on our plans to change to CSRS after the 3.2.10 upgrade.&lt;/p&gt;

&lt;p&gt;Did you start getting the errors before you upgraded the mongos instances or only after they were upgraded? &amp;#8211; The error started 10 minutes after we upgraded the mongos to 3.2.10. When we revert to 3.2.8 the error went away. &lt;/p&gt;

&lt;p&gt;Would it be possible to temporarily increase the network logging verbosity on one of the mongos instances where this problem reproduces so we can capture what exact operations were causing this timeout? &amp;#8211; I will try to reproduce the issue with &quot;network.verbosity&quot;=2 and provide you the output. &lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
Antonis&lt;/p&gt;</comment>
                            <comment id="1409945" author="kaloian.manassiev" created="Mon, 17 Oct 2016 12:16:41 +0000"  >&lt;p&gt;Hi Antonis,&lt;/p&gt;

&lt;p&gt;From the log snippet, which you have attached it seems that the errors are coming from a mongos instance - is this correct?&lt;/p&gt;

&lt;p&gt;Would it be possible to attach the logs from one such &lt;tt&gt;mongos&lt;/tt&gt; instance around the time of the errors and the logs from the &lt;tt&gt;mongod&lt;/tt&gt; instance running at &lt;tt&gt;node0-49631&lt;/tt&gt; from the same time so we can see what was happening?&lt;/p&gt;

&lt;p&gt;I also have a couple of questions:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;Are you running your config servers in SCCC mode or in CSRS?&lt;/li&gt;
	&lt;li&gt;Did you start getting the errors before you upgraded the mongos instances or only after they were upgraded? I.e., were the errors happening when the shards were running 3.2.10 and mongos at 3.2.8?&lt;/li&gt;
	&lt;li&gt;Would it be possible to temporarily increase the network logging verbosity on one of the mongos instances where this problem reproduces so we can capture what exact operations were causing this timeout?&lt;br/&gt;
The command to do this from the shell is:
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;db.setLogLevel(2, &apos;systemLog.component.network.verbosity&apos;);&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;after a few errors have been collected, you can turn the verbosity back to 0 using the following command:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;db.setLogLevel(0, &apos;systemLog.component.network.verbosity&apos;);&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Thank you in advance.&lt;/p&gt;

&lt;p&gt;Best regards,&lt;br/&gt;
-Kal.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                            <outwardlinks description="duplicates">
                                        <issuelink>
            <issuekey id="325869">SERVER-26723</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                                        </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="328331">SERVER-26859</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="144091" name="201.zip" size="107861372" author="alessandro.gherardi@yahoo.com" created="Sat, 12 Nov 2016 19:56:38 +0000"/>
                            <attachment id="144090" name="202.zip" size="17784936" author="alessandro.gherardi@yahoo.com" created="Sat, 12 Nov 2016 19:56:10 +0000"/>
                            <attachment id="144089" name="203.zip" size="35682821" author="alessandro.gherardi@yahoo.com" created="Sat, 12 Nov 2016 19:56:18 +0000"/>
                            <attachment id="142286" name="log-20161012-app0.gz" size="5628587" author="antogiann" created="Tue, 25 Oct 2016 12:59:20 +0000"/>
                            <attachment id="142287" name="log-20161012-node0-49631.gz" size="3038084" author="antogiann" created="Tue, 25 Oct 2016 12:59:20 +0000"/>
                            <attachment id="142288" name="sysbench-config" size="3712" author="antogiann" created="Tue, 25 Oct 2016 13:12:21 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>16.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[500A000000VmTHNIA3]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 17 Oct 2016 12:16:41 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        7 years, 13 weeks, 4 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>andrew.morrow@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            7 years, 13 weeks, 4 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>alessandro.gherardi@yahoo.com</customfieldvalue>
            <customfieldvalue>antogiann</customfieldvalue>
            <customfieldvalue>jonhyman</customfieldvalue>
            <customfieldvalue>kaloian.manassiev@mongodb.com</customfieldvalue>
            <customfieldvalue>royrez@microsoft.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrjt8v:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrc8vj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="1314">Sharding 2016-11-21</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrvvon:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>