<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:43:19 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-36512] replset members:  /data/WiredTigerLAS.wt grows unlimited  </title>
                <link>https://jira.mongodb.org/browse/SERVER-36512</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;rs.conf()&lt;br/&gt;
{&lt;br/&gt;
...&lt;br/&gt;
&quot;version&quot; : 560512,&lt;br/&gt;
&quot;protocolVersion&quot; : NumberLong(1),&lt;/p&gt;

&lt;p&gt;...&lt;/p&gt;

&lt;p&gt;/data/WiredTigerLAS.wt grows unlimited to 92% diskspace of /dava volume and&#160;&lt;/p&gt;

&lt;p&gt;get the primary&#160; member in idle status, can not response to read and write operation.&#160;&lt;/p&gt;

&lt;p&gt;rs.config() and rs.config() still works&lt;/p&gt;

&lt;p&gt;rs.reconfig() will pending there.&#160;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Any other info need provide please let me know&lt;/p&gt;</description>
                <environment>replica set &lt;br/&gt;
AWS EC2 </environment>
        <key id="585220">SERVER-36512</key>
            <summary>replset members:  /data/WiredTigerLAS.wt grows unlimited  </summary>
                <type id="6" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14720&amp;avatarType=issuetype">Question</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13202">Works as Designed</resolution>
                                        <assignee username="nick.brewer">Nick Brewer</assignee>
                                    <reporter username="brucezu">Bruce Zu</reporter>
                        <labels>
                    </labels>
                <created>Tue, 7 Aug 2018 20:58:05 +0000</created>
                <updated>Fri, 27 Oct 2023 13:53:36 +0000</updated>
                            <resolved>Thu, 9 Aug 2018 19:50:09 +0000</resolved>
                                    <version>3.6.6</version>
                                                                        <votes>0</votes>
                                    <watches>7</watches>
                                                                                                                <comments>
                            <comment id="1972233" author="brucezu" created="Fri, 10 Aug 2018 02:11:36 +0000"  >&lt;p&gt;Hi Nicky&lt;/p&gt;

&lt;p&gt;I am curious how can you said &quot;your replica set did not have enough members to satisfy the read concern&quot;&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/193751/193751_let+data+and+test+show+true.png&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;</comment>
                            <comment id="1972178" author="brucezu" created="Thu, 9 Aug 2018 22:59:23 +0000"  >&lt;p&gt;Hi Nick&lt;br/&gt;
I do not know why you close the issue before finding out the root reason&lt;br/&gt;
how to explain this issue. I know it is not easy to retrigger&#160;it but that does mean there is no bug there at all.&#160;&lt;/p&gt;

&lt;p&gt;old member 3.4.7(deleted) : ip-172-31-12-59&lt;br/&gt;
old member 3.4.7(deleted) : ip-172-31-20-52&lt;br/&gt;
old member 3.4.7(deleted) : ip-172-31-46-24&lt;/p&gt;

&lt;p&gt;these 3 members have been deleted on July 22, show from the log &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;a id=&quot;193746_thumb&quot; href=&quot;https://jira.mongodb.org/secure/attachment/193746/193746_old3members-deleted-July20.png&quot; title=&quot;old3members-deleted-July20.png&quot; file-preview-type=&quot;image&quot; file-preview-id=&quot;193746&quot; file-preview-title=&quot;old3members-deleted-July20.png&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/thumbnail/193746/_thumb_193746.png&quot; style=&quot;border: 0px solid black&quot; role=&quot;presentation&quot;/&gt;&lt;/a&gt;&lt;/span&gt; &lt;br/&gt;
But primary still connect them till I restart the primary node mongod service:&lt;br/&gt;
 &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;a id=&quot;193747_thumb&quot; href=&quot;https://jira.mongodb.org/secure/attachment/193747/193747_still+connect+removed+member+before+restart+mongod+service.png&quot; title=&quot;still connect removed member before restart mongod service.png&quot; file-preview-type=&quot;image&quot; file-preview-id=&quot;193747&quot; file-preview-title=&quot;still connect removed member before restart mongod service.png&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/thumbnail/193747/_thumb_193747.png&quot; style=&quot;border: 0px solid black&quot; role=&quot;presentation&quot;/&gt;&lt;/a&gt;&lt;/span&gt;&lt;/p&gt;</comment>
                            <comment id="1972063" author="brucezu" created="Thu, 9 Aug 2018 20:59:36 +0000"  >&lt;p&gt;Hi Nick,&lt;/p&gt;

&lt;p&gt;I do not think it is the root reason;&lt;br/&gt;
Firstly I think I have to list the issue in time order: &lt;br/&gt;
July 20 Afternoon:&lt;/p&gt;
&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;Three 3.4 nodes down issue happened&lt;/li&gt;
	&lt;li&gt;I removed them from replset configuration at once.&lt;br/&gt;
July 20 night&lt;/li&gt;
	&lt;li&gt;I got them back to work after clean their data and added them back to replset configure.&lt;br/&gt;
July 23, Monday&lt;/li&gt;
	&lt;li&gt;I remove them from replset configuration and terminate them&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;At that time we have 3 data bearing members and one arbiter.&lt;br/&gt;
primary : 172.31.54.204&lt;br/&gt;
arbiter : ip-172-31-5-208&lt;br/&gt;
old member 3.4.7(deleted) : ip-172-31-12-59&lt;br/&gt;
old member 3.4.7(deleted) : ip-172-31-20-52&lt;br/&gt;
old member 3.4.7(deleted) : ip-172-31-46-24&lt;br/&gt;
secondary : ip-172-31-66-130&lt;br/&gt;
secondary : ip-172-31-82-157&lt;/p&gt;

&lt;p&gt;Aug 3 &lt;br/&gt;
One secondary became unreachable :&lt;/p&gt;

&lt;p&gt;primary in idle : 172.31.54.204&lt;br/&gt;
arbiter : ip-172-31-5-208&lt;br/&gt;
sec with unreachable issue: ip-172-31-66-130&lt;br/&gt;
secondary : ip-172-31-82-157&lt;/p&gt;

&lt;p&gt;Aug 6&lt;br/&gt;
primary WiredTigerLAS.wt grows to 92% was found &lt;br/&gt;
the other normal secondary WiredTigerLAS.wt grows to 64%&lt;br/&gt;
but the primary is still is primary. not step down to secondary before I restart mongod service on the primary node.&lt;/p&gt;

&lt;p&gt;Secondly I did some test: &lt;br/&gt;
Current replset: &lt;br/&gt;
old primary now is secondary : 172.31.54.204&lt;br/&gt;
arbiter : ip-172-31-5-208&lt;br/&gt;
secondary ever with unreachable issue: ip-172-31-66-130&lt;br/&gt;
secondary (current is primarny) : ip-172-31-82-157&lt;br/&gt;
new secondary added on Aug 6 : ip-172-31-67-188&lt;/p&gt;

&lt;p&gt;I just did some test and find:&lt;br/&gt;
stop any 3 secondary members or any 2 secondary + 1 arbiter will get the primary step down to secondary.&lt;/p&gt;

&lt;p&gt;If the major does include arbiter, I test to stop 2 data bearing member, only let a primary, a secondary and an arbiter alive. keep this status for 3 hours and nothing happen to the file&lt;/p&gt;

&lt;p&gt;4.0K &lt;del&gt;rw&lt;/del&gt;------ 1 mongod mongod 4.0K Aug 9 20:27 /data/WiredTigerLAS.wt&lt;/p&gt;

&lt;p&gt;So I do not know how you figure out the root reason and without confirm with guest you close the issue.&#160;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;


&lt;p&gt;Note we use default read reference &apos;Prinamry only&apos;&lt;br/&gt;
So once the primary step down to be secondary it will end any read operation connection&lt;br/&gt;
 &lt;a href=&quot;http://paste.openstack.org/show/727769/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://paste.openstack.org/show/727769/&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;during all the process the size of WiredTigerLAS there is not any change. &lt;br/&gt;
4.0K &lt;del&gt;rw&lt;/del&gt;------ 1 mongod mongod 4.0K Aug 6 20:05 WiredTigerLAS.wt&lt;/p&gt;

&lt;p&gt;As the primary will not be primary and will not handle the read operation. &lt;br/&gt;
So how could it be possible &quot; ..your primary node was required to store an increasing amount of data in the cache&quot;&lt;br/&gt;
By the way, arbiter does not contain data but it can vote. it can be apart of the majority. &lt;br/&gt;
At least the test I did show it is not the root reason.&lt;/p&gt;

&lt;p&gt;As I mentioned in the first message. &lt;br/&gt;
When I checking the mongod.log after WiredTigerLAS.wt happen on Aug 6&lt;br/&gt;
I find the primary still trying to connect the old 3 members with v3.4.7. This should a bug&lt;br/&gt;
As in the replset configuration, they have been removed one week ago. I double checked the&lt;br/&gt;
rs.config(). they are not there. But after I restarted mongod service. the rs configuration took effect. This can be checked from the mongod.log, There is not any more message to connnect those old 3 members. This never happened on 3.4.7&lt;/p&gt;</comment>
                            <comment id="1971938" author="brucezu" created="Thu, 9 Aug 2018 19:50:00 +0000"  >&lt;p&gt;let me provide more information used to fileter out not related message in log investigation&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;old primary in idle               :    172.31.54.204&lt;/li&gt;
	&lt;li&gt;arbiter                                 : ip-172-31-5-208&lt;/li&gt;
	&lt;li&gt;old member 3.4.7(deleted) : ip-172-31-12-59&lt;/li&gt;
	&lt;li&gt;old member 3.4.7(deleted) : ip-172-31-20-52&lt;/li&gt;
	&lt;li&gt;old member 3.4.7(deleted) : ip-172-31-46-24&lt;/li&gt;
	&lt;li&gt;sec with unreachable issue: ip-172-31-66-130&lt;/li&gt;
	&lt;li&gt;secondary (new primarny)  : ip-172-31-82-157&lt;/li&gt;
	&lt;li&gt;new seccondary                  : ip-172-31-67-188&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;I also find the /data disk space size of the new member 172-31-67-188 is not the same as others.  &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;a id=&quot;193720_thumb&quot; href=&quot;https://jira.mongodb.org/secure/attachment/193720/193720_date+diskspace+is+not+same.png&quot; title=&quot;date diskspace is not same.png&quot; file-preview-type=&quot;image&quot; file-preview-id=&quot;193720&quot; file-preview-title=&quot;date diskspace is not same.png&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/thumbnail/193720/_thumb_193720.png&quot; style=&quot;border: 0px solid black&quot; role=&quot;presentation&quot;/&gt;&lt;/a&gt;&lt;/span&gt;   &lt;br/&gt;
I find this is caused by a file under /data  whose name started with collection-6, in fact there are 2 files whose name start with collection-6. &lt;br/&gt;
This wired file has more than 390 M. It is severe in the secondary with an unreachable issue happen. &lt;br/&gt;
while in new member 172-31-67-188 there is one one file with name starting with collection-6&lt;br/&gt;
  &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;a id=&quot;193722_thumb&quot; href=&quot;https://jira.mongodb.org/secure/attachment/193722/193722_wired+collection+6.png&quot; title=&quot;wired collection 6.png&quot; file-preview-type=&quot;image&quot; file-preview-id=&quot;193722&quot; file-preview-title=&quot;wired collection 6.png&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/thumbnail/193722/_thumb_193722.png&quot; style=&quot;border: 0px solid black&quot; role=&quot;presentation&quot;/&gt;&lt;/a&gt;&lt;/span&gt; &lt;br/&gt;
check the DB find it is the local. &lt;br/&gt;
 &lt;img src=&quot;https://jira.mongodb.org/images/icons/attach/noimage.png&quot; imagetext=&quot;collections are not same.png|thumbnail&quot; align=&quot;absmiddle&quot; border=&quot;0&quot; /&gt; &lt;br/&gt;
 &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;a id=&quot;193727_thumb&quot; href=&quot;https://jira.mongodb.org/secure/attachment/193727/193727_local+collections+size.png&quot; title=&quot;local collections size.png&quot; file-preview-type=&quot;image&quot; file-preview-id=&quot;193727&quot; file-preview-title=&quot;local collections size.png&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/thumbnail/193727/_thumb_193727.png&quot; style=&quot;border: 0px solid black&quot; role=&quot;presentation&quot;/&gt;&lt;/a&gt;&lt;/span&gt; &lt;br/&gt;
Hope these are helpful to find the bug&lt;/p&gt;</comment>
                            <comment id="1971875" author="nick.brewer" created="Thu, 9 Aug 2018 19:18:13 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=brucezu&quot; class=&quot;user-hover&quot; rel=&quot;brucezu&quot;&gt;brucezu&lt;/a&gt;&#160;MongoDB 3.6 enables &lt;a href=&quot;https://docs.mongodb.com/manual/release-notes/3.6/#read-concern&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;read concern &quot;majority&quot;&lt;/a&gt; - with the three 3.4 nodes down, your replica set did not have enough members to satisfy the read concern, as the arbiter does not contain data. The &lt;a href=&quot;https://docs.mongodb.com/manual/reference/read-concern-majority/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;majority&lt;/a&gt; read concern ensures that the data returned will not subsequently be rolled back, by confirming that it is acknowledged by the majority of data-bearing replica set members. &lt;/p&gt;

&lt;p&gt;Because of this, your primary node was required to store an increasing amount of data in the cache, which ultimately overflowed into the lookaside table (represented by WiredTigerLAS.wt). If you&apos;ve removed the 3.4 nodes, you should have a majority of data-bearing nodes to satisfy the read concern.&lt;/p&gt;

&lt;p&gt;-Nick&lt;/p&gt;</comment>
                            <comment id="1970976" author="brucezu" created="Wed, 8 Aug 2018 23:17:52 +0000"  >&lt;p&gt;Hi Nick,&lt;/p&gt;

&lt;p&gt;1&amp;gt; The zip file for&#160;$dbpath/diagnostic.data}}directory&#160;has been uploaded yesterday&lt;/p&gt;

&lt;p&gt;&#160; &#160; &#160; can you find it?&#160;&lt;/p&gt;

&lt;p&gt;&#160;2&amp;gt;&#160; mongod.log&#160;&#160;&lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/attachment/193640/193640_mongod_log.zip&quot; title=&quot;mongod_log.zip attached to SERVER-36512&quot;&gt;mongod_log.zip&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.mongodb.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;&#160; &#160; &#160; &#160;&lt;/p&gt;

&lt;p&gt;&#160;3&amp;gt; &quot;Can you link to the issue you&apos;re referring to?&quot;&lt;/p&gt;
&lt;blockquote&gt;&lt;p&gt;all old members except arbiter all down. the log shows it is caused by &apos;Bad value ...&apos; during the sync process. I fixed it by clean the data of old member and restart them. them restore to normal status. this issue is reported and fixed on 3.6.7 just one day before I run into this issue. Anyway, I terminated all of those old members&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;&#160; &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-35795&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;https://jira.mongodb.org/browse/SERVER-35795&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;3&amp;gt; &quot;Is the full &quot;&apos;Bad value ...&quot; message available in the mongod logs?&quot;&lt;/p&gt;

&lt;p&gt;Yeah&#160; &lt;a href=&quot;http://paste.openstack.org/show/727682/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://paste.openstack.org/show/727682/&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;you will see &quot;...caused by :: BadValue: cannot write to &apos;config.system.sessions&quot;&lt;/p&gt;

&lt;p&gt;I think that&#160; issue has no relation to the current issue&lt;/p&gt;

&lt;p&gt;Yeah&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Thank you!&lt;/p&gt;

&lt;p&gt;Bruce&lt;/p&gt;</comment>
                            <comment id="1970618" author="nick.brewer" created="Wed, 8 Aug 2018 18:06:24 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=brucezu&quot; class=&quot;user-hover&quot; rel=&quot;brucezu&quot;&gt;brucezu&lt;/a&gt; Could you upload an archive (tar or zip) of the &lt;tt&gt;$dbpath/diagnostic.data&lt;/tt&gt; directory, and the mongod.log from a mongod that experienced the unbounded WiredTigerLAS.wt growth?&lt;/p&gt;

&lt;blockquote&gt;
&lt;p&gt;all old members except arbiter all down. the log shows it is caused by &apos;Bad value ...&apos; during the sync process.  I fixed it by clean the data of old member and restart them. them restore to normal status. this issue is reported and fixed on 3.6.7 just one day before I run into this issue. Anyway, I terminated all of those old members&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Can you link to the issue you&apos;re referring to? Is the full &quot;&apos;Bad value ...&quot; message available in the mongod logs?&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Nick&lt;/p&gt;





&lt;p&gt;Thanks&lt;/p&gt;</comment>
                            <comment id="1969730" author="brucezu" created="Tue, 7 Aug 2018 21:15:02 +0000"  >&lt;p&gt;`restart mongod service` fix this issue and&#160;WiredTigerLAS.wt shrinks to 4k&#160;&lt;/p&gt;

&lt;p&gt;All data bearing member are 3.6.6&#160;&lt;/p&gt;

&lt;p&gt;Arbiter is 3.4.7&lt;/p&gt;

&lt;p&gt;It is in AWS EC2. OS:&#160; 4.14.47-56.37.amzn1.x86_64 #1 SMP Wed Jun 6 18:49:01 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux&lt;/p&gt;

&lt;p&gt;RAM 4G&lt;/p&gt;

&lt;p&gt;Mem: 4041808k total, 2399252k used&#160;&lt;/p&gt;

&lt;p&gt;db.serverCmdLineOpts().parsed.storage&lt;/p&gt;
{ &quot;dbPath&quot; : &quot;/data&quot;, &quot;journal&quot; : \{ &quot;enabled&quot; : true }
&lt;p&gt; }$ df -h&lt;br/&gt;
Filesystem Size Used Avail Use% Mounted on&lt;br/&gt;
devtmpfs 2.0G 68K 2.0G 1% /dev&lt;br/&gt;
tmpfs 2.0G 0 2.0G 0% /dev/shm&lt;br/&gt;
/dev/xvda1 9.8G 1.3G 8.4G 14% /&lt;br/&gt;
/dev/xvdb 69G 760M 65G 2% /data&lt;br/&gt;
/dev/xvdc 9.8G 323M 8.9G 4% /journal&lt;br/&gt;
/dev/xvdd 4.8G 108M 4.5G 3% /log&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;ec2-user@ip-172-31-82-157 ~&amp;#93;&lt;/span&gt;$ ls -l /data/journal/&lt;br/&gt;
total 307216&lt;br/&gt;
drwxr-xr-x 2 mongod mongod 16384 Jul 20 00:47 lost+found&lt;br/&gt;
&lt;del&gt;rw&lt;/del&gt;------ 1 mongod mongod 104857600 Aug 7 21:35 WiredTigerLog.0000000042&lt;br/&gt;
&lt;del&gt;rw&lt;/del&gt;------ 1 mongod mongod 104857600 Aug 6 20:05 WiredTigerPreplog.0000000001&lt;br/&gt;
&lt;del&gt;rw&lt;/del&gt;------ 1 mongod mongod 104857600 Aug 6 20:05 WiredTigerPreplog.0000000002&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;The replset&#160;is created with 3 data bearing node and one arbiter. all of them are 3.4.7&lt;/p&gt;

&lt;p&gt;1 week ago I added 3 new secondaries with 3.6.6 and, select&#160;a new primary with a new member by reconfiguration of the priority. An issue happened at that time after old primary become secondary and a new member becomes&#160;primary: all old members except arbiter all down. the log shows it is caused by &apos;Bad value ...&apos; during the sync process.&#160; I fixed it by clean the data of old member and restart them. them restore to normal status. this issue is reported and fixed on 3.6.7 just one day before I run into this issue. Anyway, I terminated all of those old members&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Another&#160;thing need mention:&#160; I have removed the old members from replset&#160;config. but from the mongod.log I find primary still send&#160;heartbeat to those old members.&#160;&#160;&lt;/p&gt;

&lt;p&gt;fixed it by restart mongod service.&#160;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;in 3 data bearing node and 1 arbiter:&#160;&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;&lt;tt&gt;&#160;db.version()&lt;/tt&gt;&#160;&#160;&lt;/li&gt;
	&lt;li&gt;&#160;&#160;&lt;tt&gt;rs.conf()&lt;/tt&gt;&#160;&lt;/li&gt;
	&lt;li&gt;&#160;&#160;&lt;tt&gt;rs.status()&#160;&lt;/tt&gt;&lt;/li&gt;
	&lt;li&gt;&#160;&#160;&lt;tt&gt;db.serverStatus().metrics.cursor&lt;/tt&gt;&#160;&#160;&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;result:&#160;&lt;a href=&quot;http://paste.openstack.org/show/727581/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://paste.openstack.org/show/727581/&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="193816" name="172-31-54-204 is primary before I restart it.pdf" size="44676" author="brucezu" created="Fri, 10 Aug 2018 19:05:35 +0000"/>
                            <attachment id="193716" name="Screen Shot 2018-08-09 at 12.59.09 PM.png" size="464984" author="nick.brewer" created="Thu, 9 Aug 2018 17:18:44 +0000"/>
                            <attachment id="193713" name="Screen Shot 2018-08-09 at 12.59.09 PM.png" size="464984" author="nick.brewer" created="Thu, 9 Aug 2018 17:01:41 +0000"/>
                            <attachment id="193721" name="collections is not same.png" size="266556" author="brucezu" created="Thu, 9 Aug 2018 18:40:28 +0000"/>
                            <attachment id="193720" name="date diskspace is not same.png" size="34709" author="brucezu" created="Thu, 9 Aug 2018 18:39:32 +0000"/>
                            <attachment id="193536" name="diagnostic.zip" size="138047460" author="brucezu" created="Tue, 7 Aug 2018 21:14:03 +0000"/>
                            <attachment id="193751" name="let data and test show true.png" size="112318" author="brucezu" created="Fri, 10 Aug 2018 02:11:17 +0000"/>
                            <attachment id="193727" name="local collections size.png" size="348484" author="brucezu" created="Thu, 9 Aug 2018 19:49:25 +0000"/>
                            <attachment id="193640" name="mongod_log.zip" size="13552445" author="brucezu" created="Wed, 8 Aug 2018 23:16:06 +0000"/>
                            <attachment id="193746" name="old3members-deleted-July20.png" size="472814" author="brucezu" created="Thu, 9 Aug 2018 22:56:44 +0000"/>
                            <attachment id="193747" name="still connect removed member before restart mongod service.png" size="104123" author="brucezu" created="Thu, 9 Aug 2018 22:58:17 +0000"/>
                            <attachment id="193722" name="wired collection 6.png" size="546017" author="brucezu" created="Thu, 9 Aug 2018 18:41:44 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>8.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 8 Aug 2018 01:34:54 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        5 years, 26 weeks, 6 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            5 years, 26 weeks, 6 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>brucezu</customfieldvalue>
            <customfieldvalue>nick.brewer</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hu4yvz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|htvkbb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hu4l5b:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>