<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:32:03 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-53858] Replica sets are going into recovery mode while running calls at 20k TPS</title>
                <link>https://jira.mongodb.org/browse/SERVER-53858</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Our setup is capable of running calls at 50-60k TPS and during this some of our replica sets were going into recovery mode and they couldn&apos;t sync with the primary. I know that I can recover my system by stopping mongo, deleting data sets and then starting it back on will sync with primary as per the guide &lt;a href=&quot;https://docs.mongodb.com/manual/tutorial/resync-replica-set-member/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://docs.mongodb.com/manual/tutorial/resync-replica-set-member/&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;We saw the below error message on our logs&#160;&lt;/p&gt;


&lt;p&gt;Resync&#160;is&#160;needed&#160;for&#160;secondary&#160;member&#160;SESSION-SET12:set01j2:vm01:27737&#160;this&#160;member&#160;is&#160;lagging&#160;behind&#160;by&#160;14747&#160;seconds&#160;from&#160;the&#160;primary&lt;/p&gt;

&lt;p&gt;Recovering system is not an issue but would like to know why some of our replica sets are going into recovery mode in the first place and request you to kindly assist&lt;/p&gt;

&lt;p&gt;Mongo version&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;root@vm01 ~&amp;#93;&lt;/span&gt;# rpm -qa | grep mongo&lt;br/&gt;
mongodb-org-3.6.17-1.el8.x86_64&lt;br/&gt;
mongodb-org-mongos-3.6.17-1.el8.x86_64&lt;br/&gt;
mongodb-org-tools-3.6.17-1.el8.x86_64&lt;br/&gt;
mongodb-org-server-3.6.17-1.el8.x86_64&lt;br/&gt;
mongodb-org-shell-3.6.17-1.el8.x86_64a&lt;/p&gt;

&lt;p&gt;Thanks and Regards,&lt;/p&gt;

&lt;p&gt;Azhar&lt;/p&gt;</description>
                <environment></environment>
        <key id="1593147">SERVER-53858</key>
            <summary>Replica sets are going into recovery mode while running calls at 20k TPS</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="4">Incomplete</resolution>
                                        <assignee username="dmitry.agranat@mongodb.com">Dmitry Agranat</assignee>
                                    <reporter username="rizwiazhar@gmail.com">Azhar Yousuf</reporter>
                        <labels>
                    </labels>
                <created>Mon, 18 Jan 2021 05:07:17 +0000</created>
                <updated>Wed, 24 Feb 2021 10:26:13 +0000</updated>
                            <resolved>Thu, 11 Feb 2021 09:44:40 +0000</resolved>
                                                                                        <votes>0</votes>
                                    <watches>3</watches>
                                                                                                                <comments>
                            <comment id="3632031" author="JIRAUSER1257350" created="Wed, 24 Feb 2021 10:26:13 +0000"  >&lt;p&gt;okay we&apos;ll try to hit the same issue and will collect all the logs.&#160;&lt;/p&gt;

&lt;p&gt;Thanks and Regards,&lt;br/&gt;
Azhar&lt;/p&gt;</comment>
                            <comment id="3611122" author="dmitry.agranat" created="Thu, 11 Feb 2021 09:44:17 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=rizwiazhar%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;rizwiazhar@gmail.com&quot;&gt;rizwiazhar@gmail.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;We haven&#8217;t heard back from you for some time, so I&#8217;m going to close this ticket. If this is still an issue for you, please provide additional information and we will reopen the ticket.&lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
Dima&lt;/p&gt;</comment>
                            <comment id="3586705" author="dmitry.agranat" created="Thu, 28 Jan 2021 09:14:38 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=rizwiazhar%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;rizwiazhar@gmail.com&quot;&gt;rizwiazhar@gmail.com&lt;/a&gt;, all the provided data does not cover the time of the event you&apos;ve mentioned, what we have now is some partial data a week after the event has occurred. This makes it difficult to understand what has caused some members to go into recovery mode.&lt;/p&gt;

&lt;p&gt;If you can reproduce this issue, I recommend collecting all the data mentioned here, for all members of the cluster and uploading a fresh set of data.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Dima&lt;/p&gt;</comment>
                            <comment id="3575120" author="JIRAUSER1257350" created="Thu, 21 Jan 2021 04:53:10 +0000"  >&lt;p&gt;Hi Dima,&lt;br/&gt;
 &#160; &#160; I only attached the logs which got stuck in recovery mode. And the issue occurred on&#160;2021-Jan-13 and somewhere around 6:00 to 8:00. However I could see some errors on heartbeat request at 10:00 which are shown below&#160;&lt;/p&gt;

&lt;p&gt;Also these are the name of the servers which got stuck in recovery mode&lt;/p&gt;
&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Member-4 - 27717 :&#160; - RECOVERING - vm15- ON-LINE - 16 hr - 3&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Member-3 - 27737 :&#160; - RECOVERING - vm15- ON-LINE - 1 days - 4&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Member-3 - 27717 :&#160; - RECOVERING - vm20- ON-LINE - 1 days - 2&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Member-1 - 27717 :&#160; - RECOVERING - vm22- ON-LINE - 1 days - 4&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Member-2 - 27757 :&#160; - RECOVERING - vm20- ON-LINE - 9 hr - 2&lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;


&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Thanks and Regards,&lt;/p&gt;

&lt;p&gt;Azhar&lt;/p&gt;</comment>
                            <comment id="3573273" author="dmitry.agranat" created="Wed, 20 Jan 2021 12:57:10 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=rizwiazhar%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;rizwiazhar@gmail.com&quot;&gt;rizwiazhar@gmail.com&lt;/a&gt;, I did not see any issue with the data you have uploaded. Could you please clarify:&lt;/p&gt;
&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;Where the issue occurred? What is the name of the server that went into recovery? What was the name of the Primary at that time?&lt;/li&gt;
	&lt;li&gt;When the issue occurred? Timestamps and timezone for the start and end of the event of the event&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Please note that the &lt;tt&gt;diagnostic.data&lt;/tt&gt; for &lt;tt&gt;nd5bwa5psm22va&lt;/tt&gt; is missing.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Dima&lt;/p&gt;</comment>
                            <comment id="3573045" author="JIRAUSER1257350" created="Wed, 20 Jan 2021 07:48:47 +0000"  >&lt;p&gt;Hi Dima,&#160;&lt;br/&gt;
 &#160; &#160;Our environment is quite big and we have a total of 51 sets. So currently I have uploaded only the diagnostics data from the previous uploaded affected sets which were stuck in recovery mode. Kindly let me know if these logs are sufficient enough&lt;/p&gt;

&lt;p&gt;Thanks and Regards,&lt;/p&gt;

&lt;p&gt;Azhar&lt;/p&gt;</comment>
                            <comment id="3573020" author="JIRAUSER1257350" created="Wed, 20 Jan 2021 06:27:54 +0000"  >&lt;p&gt;Hi Dima,&lt;br/&gt;
&#160; &#160;Currently I have uploaded the mongodb logs from the replica sets which were stuck in recovery state. I will collect the diagnostics data and all the replica set longs in some more time, because our setup is currently busy at the moment&#160;&lt;/p&gt;

&lt;p&gt;Thanks and Regards,&lt;/p&gt;

&lt;p&gt;Azhar&lt;/p&gt;</comment>
                            <comment id="3571724" author="dmitry.agranat" created="Tue, 19 Jan 2021 18:00:42 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=rizwiazhar%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;rizwiazhar@gmail.com&quot;&gt;rizwiazhar@gmail.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Would you please archive (tar or zip) the mongod.log files and the &lt;tt&gt;$dbpath/diagnostic.data&lt;/tt&gt; directory (the contents are described &lt;a href=&quot;https://docs.mongodb.com/manual/administration/analyzing-mongodb-performance/#full-time-diagnostic-data-capture&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;here&lt;/a&gt;) from &lt;b&gt;all members of the replica set&lt;/b&gt; and upload them to this &lt;a href=&quot;https://10gen-httpsupload.s3.amazonaws.com/upload_forms/4a1403f0-048f-430c-98a9-c10193257dda.html&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;support uploader&lt;/a&gt; location?&lt;/p&gt;

&lt;p&gt;Files uploaded to this portal are visible only to MongoDB employees and are routinely deleted after some time.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Dima&lt;/p&gt;
</comment>
                    </comments>
                    <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>8.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 19 Jan 2021 18:00:42 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        2 years, 50 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>rizwiazhar@gmail.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            2 years, 50 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>rizwiazhar@gmail.com</customfieldvalue>
            <customfieldvalue>dmitry.agranat@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hyq1jj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hybqfr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10750" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Steps To Reproduce</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>&lt;ol&gt;
	&lt;li&gt;Run calls at 20k TPS&lt;/li&gt;
	&lt;li&gt;Some replica sets will go into recovery mode&lt;/li&gt;
&lt;/ol&gt;
</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[dmitry.agranat@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hypnsv:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>