<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:16:15 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-27793] &apos;exception while parsing isMaster reply: BSONElement: bad type 101&apos; causes empty shard connection string to be written to config.shards</title>
                <link>https://jira.mongodb.org/browse/SERVER-27793</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;I have 22 shards cluster running for 2 years.&lt;/p&gt;

&lt;p&gt;Without any reason, one of shards was missing host:port info, which caused application read faliures.&lt;/p&gt;

&lt;p&gt;Can you see if it&apos;s known issue ?&lt;/p&gt;

&lt;p&gt;Mongo is running on V2.6.12.&lt;/p&gt;

&lt;p&gt;mongos&amp;gt; db.shards.find();&lt;/p&gt;
{ &quot;_id&quot; : &quot;TT-RS01&quot;, &quot;host&quot; : &quot;TT-RS01/tdb-a01.white.xxx.com:4308,tdb-m01.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS02&quot;, &quot;host&quot; : &quot;TT-RS02/tdb-a02.white.xxx.com:4308,tdb-m02.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS03&quot;, &quot;host&quot; : &quot;TT-RS03/tdb-a03.white.xxx.com:4308,tdb-m03.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS05&quot;, &quot;host&quot; : &quot;TT-RS05/tdb-a05.white.xxx.com:4308,tdb-m05.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS06&quot;, &quot;host&quot; : &quot;TT-RS06/tdb-a06.white.xxx.com:4308,tdb-m06.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS04&quot;, &quot;host&quot; : &quot;TT-RS04/tdb-a04.white.xxx.com:4308,tdb-m04.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS07&quot;, &quot;host&quot; : &quot;TT-RS07/&quot; } { &quot;_id&quot; : &quot;TT-RS08&quot;, &quot;host&quot; : &quot;TT-RS08/tdb-a08.white.xxx.com:4308,tdb-m08.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS09&quot;, &quot;host&quot; : &quot;TT-RS09/tdb-a09.white.xxx.com:4308,tdb-m09.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS10&quot;, &quot;host&quot; : &quot;TT-RS10/tdb-a10.white.xxx.com:4308,tdb-m10.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS11&quot;, &quot;host&quot; : &quot;TT-RS11/tdb-a11.white.xxx.com:4308,tdb-m11.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS12&quot;, &quot;host&quot; : &quot;TT-RS12/tdb-a12.white.xxx.com:4308,tdb-m12.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS13&quot;, &quot;host&quot; : &quot;TT-RS13/tdb-a13.white.xxx.com:4308,tdb-m13.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS14&quot;, &quot;host&quot; : &quot;TT-RS14/tdb-a14.white.xxx.com:4308,tdb-m14.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS15&quot;, &quot;host&quot; : &quot;TT-RS15/tdb-a15.white.xxx.com:4308,tdb-m15.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS16&quot;, &quot;host&quot; : &quot;TT-RS16/tdb-a16.white.xxx.com:4308,tdb-m16.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS17&quot;, &quot;host&quot; : &quot;TT-RS17/tdb-a17.white.xxx.com:4308,tdb-m17.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS18&quot;, &quot;host&quot; : &quot;TT-RS18/tdb-a18.white.xxx.com:4308,tdb-m18.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS19&quot;, &quot;host&quot; : &quot;TT-RS19/tdb-a19.white.xxx.com:4308,tdb-m19.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS20&quot;, &quot;host&quot; : &quot;TT-RS20/tdb-a20.white.xxx.com:4308,tdb-m20.white.xxx.com:4308&quot; }
&lt;p&gt;Type &quot;it&quot; for more&lt;/p&gt;

&lt;p&gt;I did manually update shard07 entry.&lt;br/&gt;
db.shards.update( &lt;/p&gt;
{ &quot;_id&quot; : &quot;TT-RS07&quot;}
&lt;p&gt; , &lt;/p&gt;
{&quot;_id&quot;: &quot;TT-RS07&quot;, &quot;host&quot;: &quot;TT-RS07/tdb-a07.white.xxx.com:4308,tdb-m07.white.xxx.com:4308&quot;}
&lt;p&gt;);&lt;br/&gt;
mongos&amp;gt; db.shards.find();&lt;/p&gt;
{ &quot;_id&quot; : &quot;TT-RS01&quot;, &quot;host&quot; : &quot;TT-RS01/tdb-a01.white.xxx.com:4308,tdb-m01.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS02&quot;, &quot;host&quot; : &quot;TT-RS02/tdb-a02.white.xxx.com:4308,tdb-m02.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS03&quot;, &quot;host&quot; : &quot;TT-RS03/tdb-a03.white.xxx.com:4308,tdb-m03.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS05&quot;, &quot;host&quot; : &quot;TT-RS05/tdb-a05.white.xxx.com:4308,tdb-m05.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS06&quot;, &quot;host&quot; : &quot;TT-RS06/tdb-a06.white.xxx.com:4308,tdb-m06.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS04&quot;, &quot;host&quot; : &quot;TT-RS04/tdb-a04.white.xxx.com:4308,tdb-m04.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS07&quot;, &quot;host&quot; : &quot;TT-RS07/tdb-a07.white.xxx.com:4308,tdb-m07.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS08&quot;, &quot;host&quot; : &quot;TT-RS08/tdb-a08.white.xxx.com:4308,tdb-m08.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS09&quot;, &quot;host&quot; : &quot;TT-RS09/tdb-a09.white.xxx.com:4308,tdb-m09.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS10&quot;, &quot;host&quot; : &quot;TT-RS10/tdb-a10.white.xxx.com:4308,tdb-m10.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS11&quot;, &quot;host&quot; : &quot;TT-RS11/tdb-a11.white.xxx.com:4308,tdb-m11.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS12&quot;, &quot;host&quot; : &quot;TT-RS12/tdb-a12.white.xxx.com:4308,tdb-m12.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS13&quot;, &quot;host&quot; : &quot;TT-RS13/tdb-a13.white.xxx.com:4308,tdb-m13.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS14&quot;, &quot;host&quot; : &quot;TT-RS14/tdb-a14.white.xxx.com:4308,tdb-m14.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS15&quot;, &quot;host&quot; : &quot;TT-RS15/tdb-a15.white.xxx.com:4308,tdb-m15.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS16&quot;, &quot;host&quot; : &quot;TT-RS16/tdb-a16.white.xxx.com:4308,tdb-m16.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS17&quot;, &quot;host&quot; : &quot;TT-RS17/tdb-a17.white.xxx.com:4308,tdb-m17.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS18&quot;, &quot;host&quot; : &quot;TT-RS18/tdb-a18.white.xxx.com:4308,tdb-m18.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS19&quot;, &quot;host&quot; : &quot;TT-RS19/tdb-a19.white.xxx.com:4308,tdb-m19.white.xxx.com:4308&quot; } { &quot;_id&quot; : &quot;TT-RS20&quot;, &quot;host&quot; : &quot;TT-RS20/tdb-a20.white.xxx.com:4308,tdb-m20.white.xxx.com:4308&quot; }
&lt;p&gt;Type &quot;it&quot; for more&lt;/p&gt;</description>
                <environment></environment>
        <key id="348142">SERVER-27793</key>
            <summary>&apos;exception while parsing isMaster reply: BSONElement: bad type 101&apos; causes empty shard connection string to be written to config.shards</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13203">Gone away</resolution>
                                        <assignee username="kaloian.manassiev@mongodb.com">Kaloian Manassiev</assignee>
                                    <reporter username="jing66wu@yahoo.com">Jing Wu</reporter>
                        <labels>
                    </labels>
                <created>Mon, 23 Jan 2017 21:31:06 +0000</created>
                <updated>Fri, 27 Oct 2023 20:44:15 +0000</updated>
                            <resolved>Fri, 10 Mar 2017 14:34:55 +0000</resolved>
                                    <version>2.6.12</version>
                    <version>3.0.14</version>
                                                    <component>MMAPv1</component>
                                        <votes>0</votes>
                                    <watches>4</watches>
                                                                                                                <comments>
                            <comment id="1520982" author="thomas.schubert" created="Fri, 10 Mar 2017 14:34:55 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=jing66wu%40yahoo.com&quot; class=&quot;user-hover&quot; rel=&quot;jing66wu@yahoo.com&quot;&gt;jing66wu@yahoo.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;As per Kaloian&apos;s last comment, I&apos;m resolving this ticket since the issue appears to be caused by a hardware defect. Please let us know if you see the error again after removing a32 from your cluster.&lt;/p&gt;

&lt;p&gt;Kind regards,&lt;br/&gt;
Thomas&lt;/p&gt;</comment>
                            <comment id="1495937" author="kaloian.manassiev" created="Thu, 9 Feb 2017 15:15:50 +0000"  >&lt;p&gt;The error messages will look the same, but I am confident that if a32 is out of the picture you will not experience it anymore.&lt;/p&gt;</comment>
                            <comment id="1495898" author="jing66wu@yahoo.com" created="Thu, 9 Feb 2017 14:48:02 +0000"  >&lt;p&gt;Kal,&lt;/p&gt;

&lt;p&gt;We are setting up monitor job to watch mongos.log to capture this issue. Since I upgrade server to V3.2.12, I am not sure if error message will change..&lt;/p&gt;

&lt;p&gt;Can you  let me know what&apos;s mongos  error messages will be look like after config.shards collection is updated with empty host:port ?&lt;/p&gt;

&lt;p&gt;Thanks,&lt;/p&gt;

&lt;p&gt;Jing&lt;/p&gt;</comment>
                            <comment id="1495002" author="kaloian.manassiev" created="Wed, 8 Feb 2017 16:19:31 +0000"  >&lt;p&gt;Thanks - this might indicate some hardware problem with a32.&lt;/p&gt;

&lt;p&gt;Please see if the problem continues happening with this mongos host taken offline. In the mean time I have filed &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-27949&quot; title=&quot;Failure to parse isMaster response in ReplicaSetMonitor can cause bad information to be written to config.shards&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-27949&quot;&gt;&lt;del&gt;SERVER-27949&lt;/del&gt;&lt;/a&gt; to ensure that one faulty mongos cannot write bad data impacting other cluster members.&lt;/p&gt;

&lt;p&gt;-Kal.&lt;/p&gt;</comment>
                            <comment id="1494962" author="jing66wu@yahoo.com" created="Wed, 8 Feb 2017 15:58:46 +0000"  >&lt;p&gt;Kal,&lt;/p&gt;

&lt;p&gt;The 3 occurances were happened on mongos on a32. The 2nd and 3rd were not on isMatser() issue, but did complain bad data, see attached mongos_a32 log.   &lt;/p&gt;

&lt;p&gt;Thanks,&lt;/p&gt;

&lt;p&gt;Jing&lt;/p&gt;</comment>
                            <comment id="1494814" author="kaloian.manassiev" created="Wed, 8 Feb 2017 13:58:21 +0000"  >&lt;p&gt;Yes, you can directly upgrade to 3.2 and continue to use the legacy 3-config server setup with MMAP V1, there is no requirement to change the storage engine to WT. Switching the config servers to WT is only required when changing the config server to use replica sets (which we highly recommend you do, but is not required).&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;... since a32 host is problem host, sa already took it out from production. Will let you know if it will happene again.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;That was another suspicion I had - that due to bad hardware, memory contents somehow get corrupted. Is it always the same &lt;tt&gt;mongos&lt;/tt&gt; which is experiencing the &quot;&lt;tt&gt;exception while parsing isMaster reply&lt;/tt&gt;&quot; error?&lt;/p&gt;

&lt;p&gt;Best regards,&lt;br/&gt;
-Kal.&lt;/p&gt;</comment>
                            <comment id="1494400" author="jing66wu@yahoo.com" created="Tue, 7 Feb 2017 21:54:57 +0000"  >&lt;p&gt;Kal,&lt;/p&gt;

&lt;p&gt;Reagarding upgrade to V3.2, can config db still using MMAPv1 engine  while the mongod uses WT engine ?&lt;br/&gt;
Documentation is not very clear about it.&lt;/p&gt;

&lt;p&gt;I plan to upgrade Mongo cluster to V3.2.12 with MMAPV1 engine instead of WT engine.  since a32 host is problem host, sa already took it out from production.   Will let you know if it will happene again.&lt;br/&gt;
Thanks,&lt;/p&gt;

&lt;p&gt;Jing&lt;/p&gt;</comment>
                            <comment id="1494377" author="kaloian.manassiev" created="Tue, 7 Feb 2017 21:33:10 +0000"  >&lt;p&gt;There is no significant difference between sharding in 2.6 and 3.0. Version 3.2 is where we did more significant refactoring and cleanup of the sharding code. I suspect that this upgrade by itself should take care of the issue you are experiencing, but even if it doesn&apos;t, like I mentioned it contains additional checks which are extremely hard to deduce from looking at logs.&lt;/p&gt;

&lt;p&gt;If you would like us to continue investigating this, you will have to upgrade to at least 3.2.12.&lt;/p&gt;

&lt;p&gt;-Kal.&lt;/p&gt;</comment>
                            <comment id="1494354" author="jing66wu@yahoo.com" created="Tue, 7 Feb 2017 21:20:36 +0000"  >&lt;p&gt;Kal,&lt;/p&gt;

&lt;p&gt;The issue on 1/20 was on V2.6.12, but 2nd and 3rd time when it happaned , the mongo clusters and  mongos  are all  on V3.0.14. &lt;br/&gt;
Please look at the logs again.  &lt;/p&gt;

&lt;p&gt;Thanks,&lt;/p&gt;

&lt;p&gt;Jing&lt;/p&gt;



</comment>
                            <comment id="1494324" author="kaloian.manassiev" created="Tue, 7 Feb 2017 21:04:19 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=jing66wu%40yahoo.com&quot; class=&quot;user-hover&quot; rel=&quot;jing66wu@yahoo.com&quot;&gt;jing66wu@yahoo.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Thanks for providing the latest logs. The information they contain is more useful and from there I can see that there was a problem parsing the &lt;tt&gt;isMaster&lt;/tt&gt; response from the shard primary and that&apos;s why it was interpreted as if there are no hosts:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-01-20T17:46:55.492-0500 [ReplicaSetMonitorWatcher] exception while parsing isMaster reply: BSONElement: bad type 101 { setName: &quot;TT-RS13&quot;, setVersion: 5, ismaster: false, secondary: true, hosts: [ &quot;tdb-m13.white.xxx.com:4308&quot;, &quot;tdb-a13.white.xxx.com:4308&quot; ], arbiters: [ &quot;itsdb-l03.white.xxx.com:27036&quot;, &quot;itsdb-m03.white.xxx.com:27036&quot; ], primary: &quot;tdb-a13.white.xxx.com:4308&quot;, me: &quot;tdb-m13.white.xxx.com:4308&quot;, maxBsonObjectSize: 16777216, maxMessageSizeBytes: 48000000, maxWriteBatchSize: 1000, localTime: new Date(1484952415476), maxWireVersion: 2, minWireVersion: 0, ok: 1.0 }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;This indicates message content corruption for some reason, but I am unable to pinpoint the cause from just the logs nor I am aware of cases where in-flight messages would get corrupted.&lt;/p&gt;

&lt;p&gt;Like I mentioned before, there have been significant changes in the replica set monitoring starting in version 3.2 and also there are tighter checks, which I am hopeful will pinpoint the place where data gets corrupted earlier so he have something better to work with. I urge you to upgrade until at least till that version and see whether the problem still reproduces.&lt;/p&gt;

&lt;p&gt;The version of mongos, which you are running is still 2.6.12 and it is &lt;a href=&quot;https://www.mongodb.com/blog/post/mongodb-2-6-end-of-life&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;no longer supported&lt;/a&gt;:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-01-20T17:07:53.401-0500 [mongosMain] MongoS version 2.6.12 starting: pid=10381 port=4356 64-bit host=its-a32.white.xxx.com (--help for usage)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Best regards,&lt;br/&gt;
-Kal.&lt;/p&gt;</comment>
                            <comment id="1493865" author="jing66wu@yahoo.com" created="Tue, 7 Feb 2017 14:51:26 +0000"  >&lt;p&gt;Kal,&lt;/p&gt;

&lt;p&gt;Can you look at upload logs and let me know what you find ?&lt;/p&gt;

&lt;p&gt;Thanks,&lt;/p&gt;

&lt;p&gt;Jing&lt;/p&gt;</comment>
                            <comment id="1493486" author="jing66wu@yahoo.com" created="Tue, 7 Feb 2017 00:27:18 +0000"  >&lt;p&gt;Kal,&lt;/p&gt;

&lt;p&gt;all  those updated happened only on  a32 host.  the rest of mongos hosts have no messages match &quot;changing host to&quot; at all.&lt;/p&gt;

&lt;p&gt;I just looked at mongos log. if it&apos;s detecting issue which certain shard unavailable , updated  metadata, why didn&apos;t it update value back ?    &lt;/p&gt;

&lt;p&gt;Please see attached mongos log.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;/p&gt;

&lt;p&gt;Jing&lt;/p&gt;</comment>
                            <comment id="1493476" author="jing66wu@yahoo.com" created="Tue, 7 Feb 2017 00:14:56 +0000"  >&lt;p&gt;Kal,&lt;/p&gt;

&lt;p&gt;I asked sa to check mongos log before. I just search myself today, I did find the metadata update msgs on one of mongos log:&lt;/p&gt;



&lt;p&gt;2017-01-20T20:31:41.098-0500 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicaSetMonitorWatcher&amp;#93;&lt;/span&gt; changing hosts to TT-RS07/ from TT-RS07/tdb-a07.white.xxx.com:4308,tdb-m07.white.xxx.com:4308&lt;br/&gt;
2017-01-25T20:08:22.981-0500 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; MaxChunkSize changing from 64MB to 1852990835MB&lt;br/&gt;
2017-02-02T23:41:26.746-0500 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicaSetMonitorWatcher&amp;#93;&lt;/span&gt; changing hosts to TT-RS13/ from TT-RS13/tdb-a13.white.xxx.com:4308,tdb-m13.white.xxx.com:4308&lt;br/&gt;
2017-02-05T04:44:40.508-0500 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicaSetMonitorWatcher&amp;#93;&lt;/span&gt; changing hosts to TT-RS04/ from TT-RS04/tdb-a04.white.xxx.com:4308,tdb-m04.white.xxx.com:4308&lt;/p&gt;



&lt;p&gt;Thanks,&lt;/p&gt;

&lt;p&gt;Jing&lt;/p&gt;</comment>
                            <comment id="1493469" author="jing66wu@yahoo.com" created="Mon, 6 Feb 2017 23:54:16 +0000"  >&lt;p&gt;Kal,&lt;/p&gt;

&lt;p&gt;Logging for error msgs are changed for V3.0.14..  what&apos;s msgs should I search for metadata updates ?  Will metadata update msgs show up in one of mongos lof or all of mongos log ?&lt;/p&gt;

&lt;p&gt;I am in process of upgrade to V3.2, have to go to V3.0.14 first , then convert to WT engine , then V3.2. It takes time for production system.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;/p&gt;

&lt;p&gt;Jing&lt;/p&gt;
</comment>
                            <comment id="1493430" author="kaloian.manassiev" created="Mon, 6 Feb 2017 22:51:32 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=jing66wu%40yahoo.com&quot; class=&quot;user-hover&quot; rel=&quot;jing66wu@yahoo.com&quot;&gt;jing66wu@yahoo.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;I am unable to pinpoint a specific piece of code or see a bug which would cause a replica set to return an empty list of nodes. Also I do not see any shard metadata updates happening in the log, which you attached.&lt;/p&gt;

&lt;p&gt;In MongoDB version 3.2 we significantly improved the replica set monitoring code, fixed race conditions and added better diagnostics. Therefore, I strongly reccomend that you upgrade to the latest version of MongoDB 3.2.&lt;/p&gt;

&lt;p&gt;Would you please upgrade to MongoDB 3.2.12 and confirm if this issue is resolved?&lt;/p&gt;

&lt;p&gt;Best regards,&lt;br/&gt;
-Kal.&lt;/p&gt;</comment>
                            <comment id="1492433" author="jing66wu@yahoo.com" created="Sun, 5 Feb 2017 17:02:14 +0000"  >&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-02-05T06:24:19.814-0500 I NETWORK  [Balancer] scoped connection to tconfigdb-m02.db.xxx.com:4307,tconfigdb-d01.db.xxx.com:4307,tconfigdb-m01.db.xxx.com:4307 not being returned to the pool&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-02-05T06:24:19.815-0500 I SHARDING [Balancer] caught exception while doing balance: Empty host component parsing HostAndPort from &quot;&quot;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-02-05T06:24:24.169-0500 W NETWORK  [conn965] db exception when initializing on TT-RS22:TT-RS22/tdb-a42.white.xxx.com:4308,tdb-m22.white.xxx.com:4308, current connection state is { state: { conn: &quot;TT-RS22/tdb-a42.white.xxx.com:4308,tdb-m22.white.xxx.com:4308&quot;, vinfo: &quot;historyDb.History @ 3290|1||55c0dbaf5b591f0b0cfcee5f&quot;, cursor: &quot;(none)&quot;, count: 0, done: false }, retryNext: false, init: false, finish: false, errored: false } :: caused by :: 9 Empty host component parsing HostAndPort from &quot;&quot;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-02-05T06:24:24.174-0500 I -        [conn965] Assertion failure _name.size() src/mongo/s/shard.h 88&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-02-05T06:24:24.183-0500 I CONTROL  [conn965]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; 0xa980a2 0xa3a8a9 0xa1e22a 0x96a831 0x9e0ed1 0x8ff15d 0x9005cd 0x908eb3 0x99de61 0x9200b7 0x9b09cc 0x99d40d 0x5e475a 0xa46f49 0x3326607aa1 0x33262e8aad&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;----- BEGIN BACKTRACE -----&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{&quot;backtrace&quot;:[{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;6980A2&quot;,&quot;s&quot;:&quot;_ZN5mongo15printStackTraceERSo&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;63A8A9&quot;,&quot;s&quot;:&quot;_ZN5mongo10logContextEPKc&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;61E22A&quot;,&quot;s&quot;:&quot;_ZN5mongo12verifyFailedEPKcS1_j&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;56A831&quot;,&quot;s&quot;:&quot;_ZNK5mongo21DBClientShardResolver15chooseWriteHostERKSsPNS_16ConnectionStringE&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;5E0ED1&quot;,&quot;s&quot;:&quot;_ZN5mongo14BatchWriteExec12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseEPNS_19BatchWriteExecStatsE&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;4FF15D&quot;,&quot;s&quot;:&quot;_ZN5mongo13ClusterWriter10shardWriteERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;5005CD&quot;,&quot;s&quot;:&quot;_ZN5mongo13ClusterWriter5writeERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;508EB3&quot;,&quot;s&quot;:&quot;_ZN5mongo15ClusterWriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;59DE61&quot;,&quot;s&quot;:&quot;_ZN5mongo7Command22execCommandClientBasicEPNS_16OperationContextEPS0_RNS_11ClientBasicEiPKcRNS_7BSONObjERNS_14BSONObjBuilderEb&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;5200B7&quot;,&quot;s&quot;:&quot;_ZN5mongo7Command20runAgainstRegisteredEPKcRNS_7BSONObjERNS_14BSONObjBuilderEi&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;5B09CC&quot;,&quot;s&quot;:&quot;_ZN5mongo8Strategy15clientCommandOpERNS_7RequestE&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;59D40D&quot;,&quot;s&quot;:&quot;_ZN5mongo7Request7processEi&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;1E475A&quot;,&quot;s&quot;:&quot;_ZN5mongo21ShardedMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;646F49&quot;,&quot;s&quot;:&quot;_ZN5mongo17PortMessageServer17handleIncomingMsgEPv&quot;},{&quot;b&quot;:&quot;3326600000&quot;,&quot;o&quot;:&quot;7AA1&quot;},{&quot;b&quot;:&quot;3326200000&quot;,&quot;o&quot;:&quot;E8AAD&quot;,&quot;s&quot;:&quot;clone&quot;}],&quot;processInfo&quot;:{ &quot;mongodbVersion&quot; : &quot;3.0.14&quot;, &quot;gitVersion&quot; : &quot;08352afcca24bfc145240a0fac9d28b978ab77f3&quot;, &quot;uname&quot; : { &quot;sysname&quot; : &quot;Linux&quot;, &quot;release&quot; : &quot;2.6.32-642.13.1.el6.x86_64&quot;, &quot;version&quot; : &quot;#1 SMP Wed Jan 11 20:56:24 UTC 2017&quot;, &quot;machine&quot; : &quot;x86_64&quot; }, &quot;somap&quot; : [ { &quot;elfType&quot; : 2, &quot;b&quot; : &quot;400000&quot;, &quot;buildId&quot; : &quot;E2E4D3C2B1703CEF484CCFE185BDA8AB474CCF52&quot; }, { &quot;b&quot; : &quot;7FFEC626E000&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;B7D50C2161B6674BD26DB5BA69B12A619488BBC5&quot; }, { &quot;path&quot; : &quot;/lib64/libpthread.so.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;B3BD4C596D72FCBE4607C86FEEC14F47B46D0DCC&quot; }, { &quot;path&quot; : &quot;/usr/lib64/libssl.so.10&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;7CE5058BE3F2AE6EBC0DF61E790C3FCED1A34C7B&quot; }, { &quot;path&quot; : &quot;/usr/lib64/libcrypto.so.10&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;096F60DC43484ECA364206125CED7D563FF9D027&quot; }, { &quot;path&quot; : &quot;/lib64/librt.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;86FE5BC1F46B8F8AA9A7A479FF991900DB93F720&quot; }, { &quot;path&quot; : &quot;/lib64/libdl.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;045D39F19533291EBD72D0EE0247F9D49BE2521E&quot; }, { &quot;path&quot; : &quot;/lib64/libm.so.6&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;B60EF3FCE5D4D2D8BAD2585D5CAAA1167B35DBFD&quot; }, { &quot;path&quot; : &quot;/lib64/libgcc_s.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;9A6E4BDFA184364D81F7DFD789474C3FB8F98A00&quot; }, { &quot;path&quot; : &quot;/lib64/libc.so.6&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;F14F3F5B55FC325FDFA7FB6F22F73CA1C6977379&quot; }, { &quot;path&quot; : &quot;/lib64/ld-linux-x86-64.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;DB04BDF55FE66BADD1081D833C17B4881E0EEDB9&quot; }, { &quot;path&quot; : &quot;/lib64/libgssapi_krb5.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;9B852585C66329AA02EFB28497E652A40F538E78&quot; }, { &quot;path&quot; : &quot;/lib64/libkrb5.so.3&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;EF3AACAFD6BF71BB861F194C1559153FB0B020E2&quot; }, { &quot;path&quot; : &quot;/lib64/libcom_err.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;152E2C18A7A2145021A8A879A01A82EE134E3946&quot; }, { &quot;path&quot; : &quot;/lib64/libk5crypto.so.3&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;DDE6774979156442185836150FC0785170F8001F&quot; }, { &quot;path&quot; : &quot;/lib64/libz.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6&quot; }, { &quot;path&quot; : &quot;/lib64/libkrb5support.so.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;A23DAFBCE170763BF1E836A8B26113F9CD20E0DA&quot; }, { &quot;path&quot; : &quot;/lib64/libkeyutils.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;AF374BAFB7F5B139A0B431D3F06D82014AFF3251&quot; }, { &quot;path&quot; : &quot;/lib64/libresolv.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;DDF6449707FD4C49DDE32A293EEE9AC218BFC460&quot; }, { &quot;path&quot; : &quot;/lib64/libselinux.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;B4576BE308DDCF7BC31F7304E4734C3D846D0236&quot; } ] }}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongos(_ZN5mongo15printStackTraceERSo+0x32) [0xa980a2]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongos(_ZN5mongo10logContextEPKc+0xE9) [0xa3a8a9]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongos(_ZN5mongo12verifyFailedEPKcS1_j+0xDA) [0xa1e22a]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongos(_ZNK5mongo21DBClientShardResolver15chooseWriteHostERKSsPNS_16ConnectionStringE+0x4B1) [0x96a831]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongos(_ZN5mongo14BatchWriteExec12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseEPNS_19BatchWriteExecStatsE+0x351) [0x9e0ed1]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongos(_ZN5mongo13ClusterWriter10shardWriteERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x30D) [0x8ff15d]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongos(_ZN5mongo13ClusterWriter5writeERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x5FD) [0x9005cd]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongos(_ZN5mongo15ClusterWriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x2C3) [0x908eb3]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongos(_ZN5mongo7Command22execCommandClientBasicEPNS_16OperationContextEPS0_RNS_11ClientBasicEiPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0x311) [0x99de61]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongos(_ZN5mongo7Command20runAgainstRegisteredEPKcRNS_7BSONObjERNS_14BSONObjBuilderEi+0x227) [0x9200b7]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongos(_ZN5mongo8Strategy15clientCommandOpERNS_7RequestE+0x1DC) [0x9b09cc]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongos(_ZN5mongo7Request7processEi+0x60D) [0x99d40d]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongos(_ZN5mongo21ShardedMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0x5A) [0x5e475a]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongos(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x339) [0xa46f49]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; libpthread.so.0(+0x7AA1) [0x3326607aa1]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; libc.so.6(clone+0x6D) [0x33262e8aad]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="1492404" author="jing66wu@yahoo.com" created="Sun, 5 Feb 2017 11:54:26 +0000"  >&lt;p&gt;Hi Kal,&lt;/p&gt;

&lt;p&gt;Current mongo version is V3.0.14.&lt;br/&gt;
It&apos;s very strange, the issue happened again this morning  on shard04, different shard again.&lt;br/&gt;
Can you find out what&apos;s root cause for this missing shards information ?&lt;/p&gt;


&lt;p&gt;Thanks,&lt;/p&gt;

&lt;p&gt;Jing&lt;/p&gt;</comment>
                            <comment id="1492168" author="jing66wu@yahoo.com" created="Sat, 4 Feb 2017 00:37:38 +0000"  >&lt;p&gt;Kal,&lt;/p&gt;

&lt;p&gt;The first time is shard07, this time is shard13.&lt;/p&gt;

&lt;p&gt;There are over 100 mongos for this cluster,  please see attched file from one of mongos.log.&lt;/p&gt;

&lt;p&gt;You can see error msg from 1/20 when shard07 was removed and 2/2 when shard13 was removed.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;/p&gt;

&lt;p&gt;Jing&lt;/p&gt;
</comment>
                            <comment id="1492136" author="kaloian.manassiev" created="Fri, 3 Feb 2017 23:07:09 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=jing66wu%40yahoo.com&quot; class=&quot;user-hover&quot; rel=&quot;jing66wu@yahoo.com&quot;&gt;jing66wu@yahoo.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Apologies for the delayed reply.&lt;/p&gt;

&lt;p&gt;Is it always the same shard, which is experiencing this problem (TT-RS07)? Have you confirmed with your SA that as part of some maintenance they are not manually modifying the &lt;tt&gt;config.shards&lt;/tt&gt; collection or that you don&apos;t have some rogue process doing that?&lt;/p&gt;

&lt;p&gt;If you are certain that the collection is not being modified outside of the sharding systems, can you please attach the mongos logs files from the time when the above assertion started showing up and also the logs from the mongods, which comprise shard &lt;b&gt;TT-RS07&lt;/b&gt;?&lt;/p&gt;

&lt;p&gt;Thanks in advance.&lt;/p&gt;

&lt;p&gt;-Kal.&lt;/p&gt;</comment>
                            <comment id="1491495" author="jing66wu@yahoo.com" created="Fri, 3 Feb 2017 08:00:25 +0000"  >&lt;p&gt;Can someone take a look again to see why it&apos;s happening ?&lt;/p&gt;

&lt;p&gt;Thanks,&lt;/p&gt;

&lt;p&gt;Jing&lt;/p&gt;</comment>
                            <comment id="1491494" author="jing66wu@yahoo.com" created="Fri, 3 Feb 2017 07:59:53 +0000"  >&lt;p&gt;2017-02-02T23:41:36.715-0500 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; scoped connection to itsconfigdb-m02.db.aol.com:4307,itsconfigdb-d01.db.aol.com:4307,itsconfigdb-m01.db.aol.com:4307 not being returned to the pool&lt;br/&gt;
2017-02-02T23:41:36.715-0500 I SHARDING &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; caught exception while doing balance: Empty host component parsing HostAndPort from &quot;&quot;&lt;br/&gt;
2017-02-02T23:41:45.245-0500 W NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;conn725&amp;#93;&lt;/span&gt; db exception when initializing on TT-RS20:TT-RS20/tdb-a20.white.xxx.com:4308,tdb-m20.white.xxx.com:4308, current connection state is { state: &lt;/p&gt;
{ conn: &quot;TT-RS20/tdb-a20.white.aol.com:4308,tdb-m20.white.xxx.com:4308&quot;, vinfo: &quot;historyDb.History @ 3285|25||55c0dbaf5b591f0b0cfcee5f&quot;, cursor: &quot;(none)&quot;, count: 0, done: false }
&lt;p&gt;, retryNext: false, init: false, finish: false, errored: false } :: caused by :: 9 Empty host component parsing HostAndPort from &quot;&quot;&lt;br/&gt;
2017-02-02T23:41:45.294-0500 I -        &lt;span class=&quot;error&quot;&gt;&amp;#91;conn725&amp;#93;&lt;/span&gt; Assertion failure _name.size() src/mongo/s/shard.h 88&lt;br/&gt;
2017-02-02T23:41:45.311-0500 I CONTROL  &lt;span class=&quot;error&quot;&gt;&amp;#91;conn725&amp;#93;&lt;/span&gt;&lt;br/&gt;
 0xa980a2 0xa3a8a9 0xa1e22a 0x96a831 0x9e0ed1 0x8ff15d 0x9005cd 0x908eb3 0x99de61 0x9200b7 0x9b09cc 0x99d40d 0x5e475a 0xa46f49 0x3326607aa1 0x33262e8aad&lt;br/&gt;
----- BEGIN BACKTRACE -----&lt;br/&gt;
{&quot;backtrace&quot;:[&lt;/p&gt;
{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;6980A2&quot;,&quot;s&quot;:&quot;_ZN5mongo15printStackTraceERSo&quot;}
&lt;p&gt;,&lt;/p&gt;
{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;63A8A9&quot;,&quot;s&quot;:&quot;_ZN5mongo10logContextEPKc&quot;}
&lt;p&gt;,&lt;/p&gt;
{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;61E22A&quot;,&quot;s&quot;:&quot;_ZN5mongo12verifyFailedEPKcS1_j&quot;}
&lt;p&gt;,&lt;/p&gt;
{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;56A831&quot;,&quot;s&quot;:&quot;_ZNK5mongo21DBClientShardResolver15chooseWriteHostERKSsPNS_16ConnectionStringE&quot;}
&lt;p&gt;,&lt;/p&gt;
{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;5E0ED1&quot;,&quot;s&quot;:&quot;_ZN5mongo14BatchWriteExec12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseEPNS_19BatchWriteExecStatsE&quot;}
&lt;p&gt;,&lt;/p&gt;
{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;4FF15D&quot;,&quot;s&quot;:&quot;_ZN5mongo13ClusterWriter10shardWriteERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE&quot;}
&lt;p&gt;,&lt;/p&gt;
{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;5005CD&quot;,&quot;s&quot;:&quot;_ZN5mongo13ClusterWriter5writeERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE&quot;}
&lt;p&gt;,&lt;/p&gt;
{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;508EB3&quot;,&quot;s&quot;:&quot;_ZN5mongo15ClusterWriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb&quot;}
&lt;p&gt;,&lt;/p&gt;
{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;59DE61&quot;,&quot;s&quot;:&quot;_ZN5mongo7Command22execCommandClientBasicEPNS_16OperationContextEPS0_RNS_11ClientBasicEiPKcRNS_7BSONObjERNS_14BSONObjBuilderEb&quot;}
&lt;p&gt;,&lt;/p&gt;
{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;5200B7&quot;,&quot;s&quot;:&quot;_ZN5mongo7Command20runAgainstRegisteredEPKcRNS_7BSONObjERNS_14BSONObjBuilderEi&quot;}
&lt;p&gt;,&lt;/p&gt;
{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;5B09CC&quot;,&quot;s&quot;:&quot;_ZN5mongo8Strategy15clientCommandOpERNS_7RequestE&quot;}
&lt;p&gt;,&lt;/p&gt;
{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;59D40D&quot;,&quot;s&quot;:&quot;_ZN5mongo7Request7processEi&quot;}
&lt;p&gt;,&lt;/p&gt;
{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;1E475A&quot;,&quot;s&quot;:&quot;_ZN5mongo21ShardedMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE&quot;}
&lt;p&gt;,&lt;/p&gt;
{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;646F49&quot;,&quot;s&quot;:&quot;_ZN5mongo17PortMessageServer17handleIncomingMsgEPv&quot;}
&lt;p&gt;,&lt;/p&gt;
{&quot;b&quot;:&quot;3326600000&quot;,&quot;o&quot;:&quot;7AA1&quot;}
&lt;p&gt;,&lt;/p&gt;
{&quot;b&quot;:&quot;3326200000&quot;,&quot;o&quot;:&quot;E8AAD&quot;,&quot;s&quot;:&quot;clone&quot;}
&lt;p&gt;],&quot;processInfo&quot;:{ &quot;mongodbVersion&quot; : &quot;3.0.14&quot;, &quot;gitVersion&quot; : &quot;08352afcca24bfc145240a0fac9d28b978ab77f3&quot;, &quot;uname&quot; : &lt;/p&gt;
{ &quot;sysname&quot; : &quot;Linux&quot;, &quot;release&quot; : &quot;2.6.32-642.13.1.el6.x86_64&quot;, &quot;version&quot; : &quot;#1 SMP Wed Jan 11 20:56:24 UTC 2017&quot;, &quot;machine&quot; : &quot;x86_64&quot; }
&lt;p&gt;, &quot;somap&quot; : [ &lt;/p&gt;
{ &quot;elfType&quot; : 2, &quot;b&quot; : &quot;400000&quot;, &quot;buildId&quot; : &quot;E2E4D3C2B1703CEF484CCFE185BDA8AB474CCF52&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;b&quot; : &quot;7FFEC626E000&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;B7D50C2161B6674BD26DB5BA69B12A619488BBC5&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/lib64/libpthread.so.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;B3BD4C596D72FCBE4607C86FEEC14F47B46D0DCC&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/usr/lib64/libssl.so.10&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;7CE5058BE3F2AE6EBC0DF61E790C3FCED1A34C7B&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/usr/lib64/libcrypto.so.10&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;096F60DC43484ECA364206125CED7D563FF9D027&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/lib64/librt.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;86FE5BC1F46B8F8AA9A7A479FF991900DB93F720&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/lib64/libdl.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;045D39F19533291EBD72D0EE0247F9D49BE2521E&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/lib64/libm.so.6&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;B60EF3FCE5D4D2D8BAD2585D5CAAA1167B35DBFD&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/lib64/libgcc_s.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;9A6E4BDFA184364D81F7DFD789474C3FB8F98A00&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/lib64/libc.so.6&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;F14F3F5B55FC325FDFA7FB6F22F73CA1C6977379&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/lib64/ld-linux-x86-64.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;DB04BDF55FE66BADD1081D833C17B4881E0EEDB9&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/lib64/libgssapi_krb5.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;9B852585C66329AA02EFB28497E652A40F538E78&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/lib64/libkrb5.so.3&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;EF3AACAFD6BF71BB861F194C1559153FB0B020E2&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/lib64/libcom_err.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;152E2C18A7A2145021A8A879A01A82EE134E3946&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/lib64/libk5crypto.so.3&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;DDE6774979156442185836150FC0785170F8001F&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/lib64/libz.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/lib64/libkrb5support.so.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;A23DAFBCE170763BF1E836A8B26113F9CD20E0DA&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/lib64/libkeyutils.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;AF374BAFB7F5B139A0B431D3F06D82014AFF3251&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/lib64/libresolv.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;DDF6449707FD4C49DDE32A293EEE9AC218BFC460&quot; }
&lt;p&gt;, &lt;/p&gt;
{ &quot;path&quot; : &quot;/lib64/libselinux.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;B4576BE308DDCF7BC31F7304E4734C3D846D0236&quot; }
&lt;p&gt; ] }}&lt;br/&gt;
 mongos(_ZN5mongo15printStackTraceERSo+0x32) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa980a2&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongos(_ZN5mongo10logContextEPKc+0xE9) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa3a8a9&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongos(_ZN5mongo12verifyFailedEPKcS1_j+0xDA) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa1e22a&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongos(_ZNK5mongo21DBClientShardResolver15chooseWriteHostERKSsPNS_16ConnectionStringE+0x4B1) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x96a831&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongos(_ZN5mongo14BatchWriteExec12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseEPNS_19BatchWriteExecStatsE+0x351) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x9e0ed1&amp;#93;&lt;/span&gt;&lt;br/&gt;
mongos(_ZN5mongo12verifyFailedEPKcS1_j+0xDA) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa1e22a&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongos(_ZNK5mongo21DBClientShardResolver15chooseWriteHostERKSsPNS_16ConnectionStringE+0x4B1) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x96a831&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongos(_ZN5mongo14BatchWriteExec12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseEPNS_19BatchWriteExecStatsE+0x351) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x9e0ed1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongos(_ZN5mongo13ClusterWriter10shardWriteERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x30D) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x8ff15d&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongos(_ZN5mongo13ClusterWriter5writeERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x5FD) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x9005cd&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongos(_ZN5mongo15ClusterWriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x2C3) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x908eb3&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongos(_ZN5mongo7Command22execCommandClientBasicEPNS_16OperationContextEPS0_RNS_11ClientBasicEiPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0x311) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x99de61&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongos(_ZN5mongo7Command20runAgainstRegisteredEPKcRNS_7BSONObjERNS_14BSONObjBuilderEi+0x227) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x9200b7&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongos(_ZN5mongo8Strategy15clientCommandOpERNS_7RequestE+0x1DC) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x9b09cc&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongos(_ZN5mongo7Request7processEi+0x60D) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x99d40d&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongos(_ZN5mongo21ShardedMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0x5A) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x5e475a&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongos(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x339) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa46f49&amp;#93;&lt;/span&gt;&lt;br/&gt;
 libpthread.so.0(+0x7AA1) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x3326607aa1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 libc.so.6(clone+0x6D) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x33262e8aad&amp;#93;&lt;/span&gt;&lt;/p&gt;</comment>
                            <comment id="1491493" author="jing66wu@yahoo.com" created="Fri, 3 Feb 2017 07:55:45 +0000"  >&lt;p&gt;I upgrade V2.612 to V3.0.14 on Monday..&lt;/p&gt;


&lt;p&gt;Today, the same issue happened again, shard13 was missing host:port without reason:&lt;/p&gt;

{ &quot;_id&quot; : &quot;TT-RS12&quot;, &quot;host&quot; : &quot;ITT-RS12/tdb-a12.white.aol.com:4308,tdb-m12.white.xxx.com:4308&quot; }
{ &quot;_id&quot; : &quot;TT-RS13&quot;, &quot;host&quot; : &quot;TT-RS13/&quot; }
{ &quot;_id&quot; : &quot;TT-RS14&quot;, &quot;host&quot; : &quot;ITT-RS14/tdb-a14.white.aol.com:4308,tdb-m13.white.xxx.com:4308&quot; }
</comment>
                            <comment id="1485131" author="jing66wu@yahoo.com" created="Wed, 25 Jan 2017 17:55:34 +0000"  >&lt;p&gt;Thanks Karl,&lt;/p&gt;

&lt;p&gt;I asked sa to search all mongos he owned,  he did not find any instances of the word &quot;changing&quot; in any of the  mongos logs . He &lt;br/&gt;
did a case-insensitive search.&lt;/p&gt;

&lt;p&gt;We didn&apos;t aware of any network issue at that time.   Each shard has 3 nodes which are in different availability zones/datacenter.   I just look at mongod log,  there was no any heatbeat faiiure messges among 3 nodes in this shard07 either.&lt;/p&gt;


&lt;p&gt;Jing&lt;/p&gt;</comment>
                            <comment id="1484923" author="kaloian.manassiev" created="Wed, 25 Jan 2017 15:12:03 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=jing66wu%40yahoo.com&quot; class=&quot;user-hover&quot; rel=&quot;jing66wu@yahoo.com&quot;&gt;jing66wu@yahoo.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;There is a component running on each mongos node, which monitors the state of the shards&apos; replica sets and whenever it notices a change, it writes the new connection string to &lt;tt&gt;config.shards&lt;/tt&gt;. It is possible that there was a period during which none of  the nodes for &lt;b&gt;TT-RS07&lt;/b&gt; were accessible and that&apos;s why it wrote an empty set of nodes.&lt;/p&gt;

&lt;p&gt;There is logging around these changes, which should start with this message: &lt;tt&gt;changing hosts to &amp;lt;new hosts&amp;gt; from &amp;lt;old hosts&amp;gt;&lt;/tt&gt;.&lt;/p&gt;

&lt;p&gt;Would it be possible to grep through the logs on all your mongos and shards and see if such message appears for TT-RS07? If you find it, do you mind attaching the relevant log file?&lt;/p&gt;

&lt;p&gt;Thanks in advance.&lt;/p&gt;

&lt;p&gt;Best regards,&lt;br/&gt;
-Kal.&lt;/p&gt;</comment>
                            <comment id="1483142" author="jing66wu@yahoo.com" created="Mon, 23 Jan 2017 21:56:50 +0000"  >&lt;p&gt;here is messages from mongos log when it happened:&lt;/p&gt;

&lt;p&gt;2017-01-20T20:31:43.980-0500 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; Assertion: 13110:HostAndPort: host is empty&lt;br/&gt;
2017-01-20T20:31:43.989-0500 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; 0xb17956 0xad1d58 0xab8e6b 0x73fe63 0x73e978 0xa29a3a 0xa2e164 0x96df19 0xabb355 0xb60be4 0x39790079d1 0x3978ce88fd&lt;br/&gt;
 /opt/local/pkg/mongodb_current/bin/mongos(_ZN5mongo15printStackTraceERSo+0x26) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xb17956&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /opt/local/pkg/mongodb_current/bin/mongos(_ZN5mongo10logContextEPKc+0x188) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xad1d58&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /opt/local/pkg/mongodb_current/bin/mongos(_ZN5mongo11msgassertedEiPKc+0xcb) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xab8e6b&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /opt/local/pkg/mongodb_current/bin/mongos(_ZN5mongo11HostAndPortC1ERKSs+0xe3) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x73fe63&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /opt/local/pkg/mongodb_current/bin/mongos(_ZN5mongo16ConnectionString12_fillServersESs+0x2b8) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x73e978&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /opt/local/pkg/mongodb_current/bin/mongos(_ZN5mongo5Shard8_setAddrERKSs+0x9a) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa29a3a&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /opt/local/pkg/mongodb_current/bin/mongos(_ZN5mongo15StaticShardInfo6reloadEv+0x594) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa2e164&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /opt/local/pkg/mongodb_current/bin/mongos(_ZN5mongo8Balancer3runEv+0x539) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x96df19&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /opt/local/pkg/mongodb_current/bin/mongos(_ZN5mongo13BackgroundJob7jobBodyEv+0x105) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xabb355&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /opt/local/pkg/mongodb_current/bin/mongos() &lt;span class=&quot;error&quot;&gt;&amp;#91;0xb60be4&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib64/libpthread.so.0() &lt;span class=&quot;error&quot;&gt;&amp;#91;0x39790079d1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib64/libc.so.6(clone+0x6d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x3978ce88fd&amp;#93;&lt;/span&gt;&lt;br/&gt;
2017-01-20T20:31:43.989-0500 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; scoped connection to tdbconfig-m02.db.xxx.com:4307,tdbconfig-d01.db.xxx.com:4307,tdbconfig-m01.db.xxx.com:4307 not being returned to the pool&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="352627">SERVER-27949</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="148788" name="mongos.log" size="18159889" author="jing66wu@yahoo.com" created="Sat, 4 Feb 2017 00:39:44 +0000"/>
                            <attachment id="148888" name="mongos_a32.log" size="17225514" author="jing66wu@yahoo.com" created="Tue, 7 Feb 2017 00:59:24 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>25.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 25 Jan 2017 15:12:03 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        6 years, 48 weeks, 5 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            6 years, 48 weeks, 5 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>jing66wu@yahoo.com</customfieldvalue>
            <customfieldvalue>kaloian.manassiev@mongodb.com</customfieldvalue>
            <customfieldvalue>kelsey.schubert@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|ht1ejz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hsu3f3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrnep3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>