<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:13:01 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-26722] router blocks and throws ExceededTimeLimit</title>
                <link>https://jira.mongodb.org/browse/SERVER-26722</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;We have updated one of our sharded cluster from v3.0.12 to v3.2.10. Since then our cluster was not operational because many operations got blocked by the the router. The corresponding log message looks like this:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2016-10-20T11:00:22.902+0200 I ASIO     [NetworkInterfaceASIO-TaskExecutorPool-3-0] Failed to connect to s559:27017 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2016-10-20T11:00:22.918+0200 I ASIO     [NetworkInterfaceASIO-TaskExecutorPool-3-0] Failed to connect to mongo-007.ipx:27017 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2016-10-20T11:00:22.920+0200 I ASIO     [NetworkInterfaceASIO-TaskExecutorPool-3-0] Failed to connect to mongo-024.ipx:27017 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2016-10-20T11:00:22.921+0200 I ASIO     [NetworkInterfaceASIO-TaskExecutorPool-3-0] Failed to connect to mongo-007.ipx:27017 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2016-10-20T11:00:22.921+0200 I ASIO     [NetworkInterfaceASIO-TaskExecutorPool-3-0] Failed to connect to mongo-024.ipx:27017 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;We can reproduce the issue at any time just by executing a findOne through the router several times:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;for(x=0;x&amp;lt;1000;x++){db.offer.find({&quot;_id&quot; : NumberLong(&quot;5672494983&quot;)}).forEach(function(u){printjson(u)});print(x)}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;It blocks after a few findOne&apos;s already.&lt;br/&gt;
If we execute the same code on the shard where the document is located then there is no blocking at all.&lt;/p&gt;

&lt;p&gt;We found out that mongodb router v3.0.12 doesn&apos;t have this problem. This is why we downgraded all our routers to v3.0.12 even though the rest of the cluster (mongod&apos;s) is running v3.2.10.&lt;/p&gt;

&lt;p&gt;Please see attached the log file from the router. &lt;br/&gt;
Please see also 3 monitoring screenshots of the router TCP-sockets. As you can see, tcp_tw (tcp_timeWait) is much higher for v3.2.10 than for v3.0.12.&lt;/p&gt;</description>
                <environment></environment>
        <key id="325826">SERVER-26722</key>
            <summary>router blocks and throws ExceededTimeLimit</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="3">Duplicate</resolution>
                                        <assignee username="mira.carey@mongodb.com">Mira Carey</assignee>
                                    <reporter username="kay.agahd@idealo.de">Kay Agahd</reporter>
                        <labels>
                    </labels>
                <created>Fri, 21 Oct 2016 10:16:32 +0000</created>
                <updated>Mon, 8 Jan 2024 15:23:09 +0000</updated>
                            <resolved>Thu, 8 Dec 2016 18:08:38 +0000</resolved>
                                    <version>3.2.10</version>
                                                    <component>Querying</component>
                    <component>Sharding</component>
                                        <votes>2</votes>
                                    <watches>17</watches>
                                                                                                                <comments>
                            <comment id="1540948" author="kay.agahd@idealo.de" created="Tue, 4 Apr 2017 23:06:50 +0000"  >&lt;p&gt;This issue is &lt;b&gt;not&lt;/b&gt; resolved. We can reproduce it in v3.2.12. &lt;br/&gt;
The attached screenshot shows that tcp_tw (tcp_timewait) ist still very high. Our cluster is running for only 8 hours. During these 8 hours the logs of only one mongos counts already over 200000 ExceededTimeLimit erros, as for example:&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-04-05T00:57:59.767+0200 I ASIO     [NetworkInterfaceASIO-TaskExecutorPool-3-0] Failed to connect to mongo-070.db00.pro05.eu.idealo.com:27017 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-04-05T00:57:59.767+0200 I ASIO     [NetworkInterfaceASIO-TaskExecutorPool-3-0] Failed to connect to mongo-073.db00.pro05.eu.idealo.com:27017 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-04-05T00:57:59.767+0200 I ASIO     [NetworkInterfaceASIO-TaskExecutorPool-3-0] Failed to connect to mongo-067.db00.pro05.eu.idealo.com:27017 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-04-05T00:57:59.767+0200 I ASIO     [NetworkInterfaceASIO-TaskExecutorPool-3-0] Failed to connect to mongo-070.db00.pro05.eu.idealo.com:27017 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-04-05T00:57:59.767+0200 I ASIO     [NetworkInterfaceASIO-TaskExecutorPool-3-0] Failed to connect to mongo-067.db00.pro05.eu.idealo.com:27017 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-04-05T00:57:59.767+0200 I ASIO     [NetworkInterfaceASIO-TaskExecutorPool-3-0] Failed to connect to mongo-073.db00.pro05.eu.idealo.com:27017 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-04-05T00:57:59.768+0200 I ASIO     [NetworkInterfaceASIO-TaskExecutorPool-3-0] Failed to connect to mongo-067.db00.pro05.eu.idealo.com:27017 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-04-05T00:57:59.768+0200 I ASIO     [NetworkInterfaceASIO-TaskExecutorPool-3-0] Failed to connect to mongo-073.db00.pro05.eu.idealo.com:27017 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-04-05T00:57:59.768+0200 I ASIO     [NetworkInterfaceASIO-TaskExecutorPool-3-0] Failed to connect to mongo-066.db00.pro05.eu.idealo.com:27017 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-04-05T00:57:59.768+0200 I ASIO     [NetworkInterfaceASIO-TaskExecutorPool-3-0] Failed to connect to mongo-070.db00.pro05.eu.idealo.com:27017 - ExceededTimeLimit: Operation timed out&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt; &lt;br/&gt;
We observe this behaviour independent on whether the query uses the shardkey or not. In all cases the queried field is indexed. &lt;/p&gt;

&lt;p&gt;A &lt;b&gt;downgrade&lt;/b&gt; of the routers to v3.0.12 ist not possible because our configservers are running as replicaset instead of a mirrored set. &lt;br/&gt;
An &lt;b&gt;upgrade&lt;/b&gt; of the routers to v3.4.3 is not possible because &quot;Version 3.4 mongos instances cannot connect to earlier versions of mongod instances.&quot;&lt;br/&gt;
&lt;a href=&quot;https://docs.mongodb.com/manual/release-notes/3.4-compatibility/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://docs.mongodb.com/manual/release-notes/3.4-compatibility/&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Please advise.&lt;/p&gt;

&lt;p&gt;PS: Since this ticket has been closed already, we&apos;ve created a new ticket, please see: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-28629&quot; title=&quot;router blocks and throws ExceededTimeLimit&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-28629&quot;&gt;&lt;del&gt;SERVER-28629&lt;/del&gt;&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1452577" author="jason.carey" created="Thu, 8 Dec 2016 18:09:24 +0000"  >&lt;p&gt;This is fixed in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-27232&quot; title=&quot;Refresh and Setup timeouts in the ASIO connpool can prematurely time out an operation&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-27232&quot;&gt;&lt;del&gt;SERVER-27232&lt;/del&gt;&lt;/a&gt; and will be backported as that ticket number&lt;/p&gt;</comment>
                            <comment id="1447041" author="jason.carey" created="Thu, 1 Dec 2016 17:22:26 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=antogiann&quot; class=&quot;user-hover&quot; rel=&quot;antogiann&quot;&gt;antogiann&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;That&apos;s correct, ShardingTaskExecutorPoolRefreshTimeoutMS is the setparameter I&apos;m referring to.&lt;/p&gt;

&lt;p&gt;The default value is 20000 (I.e. 20seconds * 1000 ms in a second).  You&apos;ll want to increase that to a value long enough that only a very small number of connections time out.  That particular value will depend on your network and it&apos;s responsiveness under periods of load.&lt;/p&gt;

&lt;p&gt;I&apos;d suggest trying 60000 (1 minute) and ramping up from there until the behavior begins to clear up.  Note that if you have user operations with timeouts, those may trigger.  I.e. if you set a maxTimeMS on a query of 30 seconds, and occasionally it takes 1 minute to connect, those operations will still fail.&lt;/p&gt;</comment>
                            <comment id="1446998" author="jonhyman" created="Thu, 1 Dec 2016 16:39:52 +0000"  >&lt;p&gt;Great, thanks.&lt;/p&gt;</comment>
                            <comment id="1446991" author="jason.carey" created="Thu, 1 Dec 2016 16:30:29 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=jon%40appboy.com&quot; class=&quot;user-hover&quot; rel=&quot;jon@appboy.com&quot;&gt;jon@appboy.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;I&apos;m definitely considering this for 3.2 and 3.4, but haven&apos;t had a chance to run down which specific minor versions we&apos;ll target with our release team.  I&apos;ll apply specific fix versions as those targets get negotiated&lt;/p&gt;</comment>
                            <comment id="1446804" author="antogiann" created="Thu, 1 Dec 2016 14:07:28 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=mira.carey%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;mira.carey@mongodb.com&quot;&gt;mira.carey@mongodb.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;I am willing to test with an increased &quot;ShardingTaskExecutorPoolRefreshTimeoutMS&quot; on an environment that I have successfully reproduce the error multiple times. &lt;/p&gt;

&lt;p&gt;I believe ShardingTaskExecutorPoolRefreshTimeoutMS is exposed in 3.2.11 on the mongos configuration file (please confirm):&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;setParameter:&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; ShardingTaskExecutorPoolRefreshTimeoutMS: &amp;lt;&lt;/span&gt;&lt;span style=&quot;color: #006699; font-weight: bold; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;value&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&amp;gt;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;Can you recommend a starting value for ShardingTaskExecutorPoolRefreshTimeoutMS ? &lt;/p&gt;

&lt;p&gt;Thanks&lt;br/&gt;
Ant&lt;/p&gt;</comment>
                            <comment id="1446418" author="jonhyman" created="Wed, 30 Nov 2016 23:08:58 +0000"  >&lt;p&gt;I see that you set this as 3.5 Required. This issue occurred in 3.2.10 is there no way that you can back port this to 3.2.x? We&apos;re currently stuck on 3.2.8, which has mongoS segfaults, because we were forced to downgrade back after this issue immediately wrecked our application after upgrading to 3.2.10; but we&apos;re going to have to wait until 3.6 comes out before this is fixed? I may just not be understanding your Fix Version tag.&lt;/p&gt;</comment>
                            <comment id="1446399" author="jason.carey" created="Wed, 30 Nov 2016 22:53:30 +0000"  >&lt;p&gt;After some investigation, I&apos;ve opened &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-27232&quot; title=&quot;Refresh and Setup timeouts in the ASIO connpool can prematurely time out an operation&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-27232&quot;&gt;&lt;del&gt;SERVER-27232&lt;/del&gt;&lt;/a&gt;, which definitely involves early trigger of operation timeouts.&lt;/p&gt;

&lt;p&gt;The problem is that we have timeouts on connects (which aren&apos;t directly linked to operations) which, when they trigger, propagate as timeouts to user operations.  And it triggers for all connections currently in the pool.&lt;/p&gt;

&lt;p&gt;So if you&apos;re ever in a situation where you can see connections timeouts (default 20 seconds), you&apos;ll see waves of operation timeouts as a result.&lt;/p&gt;

&lt;p&gt;I believe increasing ShardingTaskExecutorPoolRefreshTimeoutMS to a higher value may be a viable work around.&lt;/p&gt;</comment>
                            <comment id="1443975" author="thomas.schubert" created="Mon, 28 Nov 2016 22:26:34 +0000"  >&lt;p&gt;Hi all,&lt;/p&gt;

&lt;p&gt;We&apos;re continuing to investigate this behavior and making progress towards reproducing this issue. Please note that this issue is distinct from &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-26723&quot; title=&quot;Mongos stalls for even simple queries&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-26723&quot;&gt;&lt;del&gt;SERVER-26723&lt;/del&gt;&lt;/a&gt;, which we have determined to be caused by a bug in glibc.&lt;/p&gt;

&lt;p&gt;Please continue to watch this ticket for updates.&lt;/p&gt;

&lt;p&gt;Kind regards,&lt;br/&gt;
Thomas&lt;/p&gt;</comment>
                            <comment id="1432585" author="kay.agahd@idealo.de" created="Sat, 12 Nov 2016 08:03:34 +0000"  >&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;09:02:26&amp;#93;&lt;/span&gt;root@offerstore-en-router-03.ipx /home/admin# uname -a&lt;br/&gt;
Linux offerstore-en-router-03 3.2.0-4-amd64 #1 SMP Debian 3.2.60-1+deb7u3 x86_64 GNU/Linux&lt;/p&gt;</comment>
                            <comment id="1432330" author="kaloian.manassiev" created="Fri, 11 Nov 2016 20:38:33 +0000"  >&lt;p&gt;Thanks &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=agherardi&quot; class=&quot;user-hover&quot; rel=&quot;agherardi&quot;&gt;agherardi&lt;/a&gt;. In your case you are definitely not hitting the glibc deadlock problem. Per our &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-26654?focusedCommentId=1431271&amp;amp;page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-1431271&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;previous conversation&lt;/a&gt;, please if possible attach the mongos logs with increased verbosity.&lt;/p&gt;

&lt;p&gt;Thanks again for your help.&lt;/p&gt;

&lt;p&gt;-Kal.&lt;/p&gt;</comment>
                            <comment id="1432234" author="alessandro.gherardi@yahoo.com" created="Fri, 11 Nov 2016 19:22:34 +0000"  >&lt;p&gt;We use Windows.&lt;/p&gt;</comment>
                            <comment id="1432129" author="antogiann" created="Fri, 11 Nov 2016 17:41:57 +0000"  >&lt;p&gt;Linux HOSTNAME 2.6.32-042stab110.1 #1 SMP Fri Jul 3 16:58:07 MSK 2015 x86_64 x86_64 x86_64 GNU/Linux&lt;/p&gt;</comment>
                            <comment id="1432107" author="royrez@microsoft.com" created="Fri, 11 Nov 2016 17:23:23 +0000"  >&lt;p&gt;Linux HOSTNAME 4.4.0-45-generic #66~14.04.1-Ubuntu SMP Wed Oct 19 15:05:38 UTC 2016 x86_64 x86_64 x86_64 GNU/Linux&lt;/p&gt;</comment>
                            <comment id="1432076" author="kaloian.manassiev" created="Fri, 11 Nov 2016 16:52:54 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=kay.agahd%40idealo.de&quot; class=&quot;user-hover&quot; rel=&quot;kay.agahd@idealo.de&quot;&gt;kay.agahd@idealo.de&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=agherardi&quot; class=&quot;user-hover&quot; rel=&quot;agherardi&quot;&gt;agherardi&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=royrez%40microsoft.com&quot; class=&quot;user-hover&quot; rel=&quot;royrez@microsoft.com&quot;&gt;royrez@microsoft.com&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=antogiann&quot; class=&quot;user-hover&quot; rel=&quot;antogiann&quot;&gt;antogiann&lt;/a&gt; - We are working on a hypothesis that this could be a bug in glibc. See &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-26723?focusedCommentId=1432067&amp;amp;page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-1432067&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;this&lt;/a&gt; comment in the related ticket for more detail.&lt;/p&gt;

&lt;p&gt;Would it be possible to run &quot;&lt;tt&gt;uname -a&lt;/tt&gt;&quot; on one of the machines where this problem occurs and include the output so we have a few samples to look at?&lt;/p&gt;

&lt;p&gt;Thank you in advance.&lt;/p&gt;

&lt;p&gt;-Kal.&lt;/p&gt;</comment>
                            <comment id="1431096" author="royrez@microsoft.com" created="Thu, 10 Nov 2016 15:33:15 +0000"  >&lt;p&gt;We have the same issue with 3.2.10.&lt;/p&gt;</comment>
                            <comment id="1430558" author="alessandro.gherardi@yahoo.com" created="Wed, 9 Nov 2016 21:02:47 +0000"  >&lt;p&gt;We&apos;re seeing the same issue when we upgraded from 3.0.4 to 3.2.10. We just tried 3.2.11-rc0 - which supposedly has the patch for &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-26859&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;https://jira.mongodb.org/browse/SERVER-26859&lt;/a&gt; - but no luck.&lt;/p&gt;

&lt;p&gt;We use CSRS.&lt;/p&gt;</comment>
                            <comment id="1423158" author="kay.agahd@idealo.de" created="Tue, 1 Nov 2016 17:31:43 +0000"  >&lt;p&gt;Good to hear, thank you!&lt;/p&gt;</comment>
                            <comment id="1423029" author="kaloian.manassiev" created="Tue, 1 Nov 2016 15:54:02 +0000"  >&lt;p&gt;Thanks &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=kay.agahd%40idealo.de&quot; class=&quot;user-hover&quot; rel=&quot;kay.agahd@idealo.de&quot;&gt;kay.agahd@idealo.de&lt;/a&gt; for the information - the original logs and your note about the increased connections in &lt;tt&gt;tcp_timeWait&lt;/tt&gt; state was really helpful.&lt;/p&gt;

&lt;p&gt;We now have a sense of what is going on and the problem is described in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-26859&quot; title=&quot;AsyncResultsMerger replica set retargeting may block the ASIO callback threads&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-26859&quot;&gt;&lt;del&gt;SERVER-26859&lt;/del&gt;&lt;/a&gt;. We are now working on getting it fixed and ported to 3.2. Please watch this ticket for further updates.&lt;/p&gt;</comment>
                            <comment id="1417856" author="kay.agahd@idealo.de" created="Wed, 26 Oct 2016 08:18:19 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=kaloian.manassiev&quot; class=&quot;user-hover&quot; rel=&quot;kaloian.manassiev&quot;&gt;kaloian.manassiev&lt;/a&gt;, I tried to reproduce the issue for over 30 minutes. Up to 10 clients executed in parallel ten thousands of finds in a loop but even if some finds could took up to 10 or even 20 seconds, it never got blocked like on October 20.&lt;br/&gt;
Nevertheless I attach you the router log set to log level 1. Btw. tcp_timeWait went up again (see screen shot attached).&lt;/p&gt;</comment>
                            <comment id="1417825" author="kay.agahd@idealo.de" created="Wed, 26 Oct 2016 06:35:05 +0000"  >&lt;p&gt;Thanks &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=kaloian.manassiev&quot; class=&quot;user-hover&quot; rel=&quot;kaloian.manassiev&quot;&gt;kaloian.manassiev&lt;/a&gt;, I&apos;ll try to install a v3.2.10 router, reproduce the issue and upload the extended router logs today.&lt;/p&gt;</comment>
                            <comment id="1417512" author="kaloian.manassiev" created="Tue, 25 Oct 2016 21:47:20 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=kay.agahd%40idealo.de&quot; class=&quot;user-hover&quot; rel=&quot;kay.agahd@idealo.de&quot;&gt;kay.agahd@idealo.de&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Apologies for the delayed reply and thank you for providing us with the mongos logs.&lt;/p&gt;

&lt;p&gt;One hypothesis that we are working on is that we have code which performs blocking I/O work in a asynchronous network worker thread. The &lt;a href=&quot;https://github.com/mongodb/mongo/blob/ce0f41c5ace60ba89e55c5244cdaf79c5d78c503/src/mongo/s/query/async_results_merger.cpp#L444&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;code in question&lt;/a&gt; would be exercised only if a network connection establishment fails for some reason, but once it happens it could block subsequent connection establishment or network calls which happen to use the same thread.&lt;/p&gt;

&lt;p&gt;The fact that it happens during a find operation makes this hypothesis even more likely. Unfortunately we only log this information at log level 1 so it is not present in these logs.&lt;/p&gt;

&lt;p&gt;Would it be possible that you perform the mongos upgrade again and run the repro with log level 1 enabled on mongos so we can see whether retries are actually happening?&lt;/p&gt;

&lt;p&gt;Thanks in advance.&lt;/p&gt;

&lt;p&gt;Best regards,&lt;br/&gt;
-Kal.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                            <outwardlinks description="duplicates">
                                        <issuelink>
            <issuekey id="335282">SERVER-27232</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                                        </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="325869">SERVER-26723</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="328331">SERVER-26859</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="153144" name="fr-11_tcpwait.jpg" size="176722" author="kay.agahd@idealo.de" created="Tue, 4 Apr 2017 23:07:33 +0000"/>
                            <attachment id="153145" name="fr-11_tcpwaitOnly.jpg" size="172606" author="kay.agahd@idealo.de" created="Tue, 4 Apr 2017 23:07:33 +0000"/>
                            <attachment id="142067" name="offerstore-en-router-03.ipx.2016-10-20.log.tgz" size="9921617" author="kay.agahd@idealo.de" created="Fri, 21 Oct 2016 10:16:32 +0000"/>
                            <attachment id="142451" name="offerstore-en-router-03.ipx.2016-10-26.log.tgz" size="3923751" author="kay.agahd@idealo.de" created="Wed, 26 Oct 2016 08:18:54 +0000"/>
                            <attachment id="142068" name="offerstore-en-router-03_afterDowngradeToV3.0.10.png" size="206231" author="kay.agahd@idealo.de" created="Fri, 21 Oct 2016 10:16:33 +0000"/>
                            <attachment id="142069" name="offerstore-en-router-03_afterUpgradeToV3.2.10.png" size="205821" author="kay.agahd@idealo.de" created="Fri, 21 Oct 2016 10:16:33 +0000"/>
                            <attachment id="142070" name="offerstore-en-router-03_beforeUpgrade.png" size="207056" author="kay.agahd@idealo.de" created="Fri, 21 Oct 2016 10:16:33 +0000"/>
                            <attachment id="142452" name="offerstore-en-router-03_testWithV3.2.10.png" size="237583" author="kay.agahd@idealo.de" created="Wed, 26 Oct 2016 08:18:54 +0000"/>
                            <attachment id="144909" name="v3.2.11 - 201.zip" size="107861372" author="kaloian.manassiev@mongodb.com" created="Tue, 29 Nov 2016 17:01:21 +0000"/>
                            <attachment id="144908" name="v3.2.11 - 202.zip" size="17784936" author="kaloian.manassiev@mongodb.com" created="Tue, 29 Nov 2016 17:01:16 +0000"/>
                            <attachment id="144907" name="v3.2.11 - 203.zip" size="35682821" author="kaloian.manassiev@mongodb.com" created="Tue, 29 Nov 2016 17:01:18 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>22.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[500A000000VmTHNIA3]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 25 Oct 2016 18:55:46 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        6 years, 45 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            6 years, 45 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>alessandro.gherardi@yahoo.com</customfieldvalue>
            <customfieldvalue>antogiann</customfieldvalue>
            <customfieldvalue>jonhyman</customfieldvalue>
            <customfieldvalue>kaloian.manassiev@mongodb.com</customfieldvalue>
            <customfieldvalue>kay.agahd@idealo.de</customfieldvalue>
            <customfieldvalue>kelsey.schubert@mongodb.com</customfieldvalue>
            <customfieldvalue>mira.carey@mongodb.com</customfieldvalue>
            <customfieldvalue>royrez@microsoft.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrjswn:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrc213:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="1314">Sharding 2016-11-21</customfieldvalue>
    <customfieldvalue id="1315">Sharding 2016-12-12</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrrvu7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>