<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:31:44 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-53738] Flow control is engaged and the sustainer point is not moving. Please check the health of all secondaries</title>
                <link>https://jira.mongodb.org/browse/SERVER-53738</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Hello Team,&lt;br/&gt;
recently we had faced an issue on our production system on 8th Jan 2021 between &lt;span class=&quot;error&quot;&gt;&amp;#91;16:20-16:40 IST&amp;#93;&lt;/span&gt;&#160; wherein one of our mongod&lt;span class=&quot;error&quot;&gt;&amp;#91;Primary&amp;#93;&lt;/span&gt;&#160; instance stop responding version we are using is&#160;&lt;b&gt;4.2.8&lt;/b&gt;.&lt;br/&gt;
we were able to login to &lt;b&gt;primary&lt;/b&gt; instance but&#160; it was getting stuck on&#160;&lt;br/&gt;
&lt;b&gt;show dbs/any command&lt;/b&gt;&#160;&#160; there was no re-index/index building going on which can cause the lock where we checked the logs and we found something&#160;&lt;br/&gt;
&lt;b&gt;Flow control is engaged and the sustainer point is not moving. Please check the health of all secondaries&lt;/b&gt;&lt;br/&gt;
&lt;b&gt;Unable to gather storage statistics for a slow operation due to lock aquire timeout&lt;/b&gt;&lt;br/&gt;
it remains for almost 10-15 mins in that state post which everything goes healthy.&lt;br/&gt;
can you kindly throw some light what could be causing this issue and let us know anything required from our side.&lt;/p&gt;</description>
                <environment></environment>
        <key id="1588832">SERVER-53738</key>
            <summary>Flow control is engaged and the sustainer point is not moving. Please check the health of all secondaries</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="dmitry.agranat@mongodb.com">Dmitry Agranat</assignee>
                                    <reporter username="sourabh.ghosh@visiblealpha.com">Sourabh Ghosh</reporter>
                        <labels>
                    </labels>
                <created>Wed, 13 Jan 2021 12:50:09 +0000</created>
                <updated>Thu, 11 Feb 2021 09:42:02 +0000</updated>
                            <resolved>Thu, 11 Feb 2021 09:42:02 +0000</resolved>
                                    <version>4.2.8</version>
                                                                        <votes>0</votes>
                                    <watches>7</watches>
                                                                                                                <comments>
                            <comment id="3611119" author="dmitry.agranat" created="Thu, 11 Feb 2021 09:41:42 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sourabh.ghosh%40visiblealpha.com&quot; class=&quot;user-hover&quot; rel=&quot;sourabh.ghosh@visiblealpha.com&quot;&gt;sourabh.ghosh@visiblealpha.com&lt;/a&gt;, since it&apos;s been over a month since the original issue was reported, I will go ahead and close this case. If this re-occurs, please collect the requested information and open a new ticket and we&apos;ll be happy to investigate.&lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
Dima&lt;/p&gt;</comment>
                            <comment id="3584293" author="JIRAUSER1258112" created="Wed, 27 Jan 2021 05:58:24 +0000"  >&lt;p&gt;Thanks Dimitry for giving your time and info.&lt;/p&gt;</comment>
                            <comment id="3582470" author="dmitry.agranat" created="Tue, 26 Jan 2021 14:20:30 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sourabh.ghosh%40visiblealpha.com&quot; class=&quot;user-hover&quot; rel=&quot;sourabh.ghosh@visiblealpha.com&quot;&gt;sourabh.ghosh@visiblealpha.com&lt;/a&gt;, the affected Primary was responding from the networking perspective. The condition for an election &quot;the secondary members losing connectivity to the primary for more than the configured timeout (10 seconds by default)&quot; was not met.&lt;/p&gt;

&lt;p&gt;There is no need to enable more verbose logging at this time.&lt;/p&gt;
</comment>
                            <comment id="3577496" author="JIRAUSER1258112" created="Fri, 22 Jan 2021 13:29:13 +0000"  >&lt;p&gt;Yes Dimitry,&lt;br/&gt;
 This occured for the very first time where in Primary was not responding&#160; but again the question remains suppose if it is not responding so one of secondary should be get promoted to primary(there has tobe election take place in such scenario).&lt;/p&gt;

&lt;p&gt;and Yes we will get more info on collecting the currentOp and lockinfo details going forward if it occurs and one more thing to know should we enable or &quot;increase the log verbosity level as a whole or any particular component level&quot; to capture log more in details ? any drawbacks because we have massive write and read intensive workload.&#160;&lt;/p&gt;</comment>
                            <comment id="3575452" author="dmitry.agranat" created="Thu, 21 Jan 2021 13:44:23 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sourabh.ghosh%40visiblealpha.com&quot; class=&quot;user-hover&quot; rel=&quot;sourabh.ghosh@visiblealpha.com&quot;&gt;sourabh.ghosh@visiblealpha.com&lt;/a&gt;, after analyzing the provided data, the event that you&apos;ve reported might have happened due to some operation holding an intent lock for an extended period without yielding (&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-53180&quot; title=&quot;Log each operation that holds a lock for an extended period of time without yielding&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-53180&quot;&gt;&lt;del&gt;SERVER-53180&lt;/del&gt;&lt;/a&gt; would have been useful in this situation). It is not obvious what that operation was during this event as several significant operations were executed at the same time.&lt;/p&gt;

&lt;p&gt;The blocking event appears to have started coinciding with a &lt;tt&gt;createIndexes&lt;/tt&gt;, &lt;tt&gt;catalogCache&lt;/tt&gt; refresh and a &lt;tt&gt;recvChunk&lt;/tt&gt; operations. This might be due to operations on one collection being blocked waiting for an incremental refresh that in turn is not progressing, or is progressing slowly. However, due to not enough information, we cannot determine this at this point.&lt;/p&gt;

&lt;p&gt;It will be helpful to collect more data the next time this happens. Specifically, collecting &lt;tt&gt;currentOp&lt;/tt&gt; and &lt;tt&gt;lockInfo&lt;/tt&gt; during such an event would provide us with more information about what databases and collections the operations are trying to take locks on and what locks they hold. If this is a reoccurring event, you could put in place a monitor to collect &lt;tt&gt;currentOp&lt;/tt&gt; and &lt;tt&gt;lockInfo&lt;/tt&gt; periodically, say every 10 or 30 sec. You can archive, rotate and delete previously collected data if it takes days to reproduce. All we need to save is the one covering the reported event.&lt;/p&gt;

&lt;p&gt;Was this a one-time event or does this occur periodically?&lt;/p&gt;</comment>
                            <comment id="3572978" author="JIRAUSER1258112" created="Wed, 20 Jan 2021 05:13:23 +0000"  >&lt;p&gt;Thanks Dimitry for the update ...&lt;/p&gt;</comment>
                            <comment id="3571704" author="dmitry.agranat" created="Tue, 19 Jan 2021 17:51:39 +0000"  >&lt;p&gt;Thanks &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sourabh.ghosh%40visiblealpha.com&quot; class=&quot;user-hover&quot; rel=&quot;sourabh.ghosh@visiblealpha.com&quot;&gt;sourabh.ghosh@visiblealpha.com&lt;/a&gt;, I am still investigating this and will have an update for you later this week&lt;/p&gt;</comment>
                            <comment id="3569648" author="JIRAUSER1258112" created="Mon, 18 Jan 2021 14:28:36 +0000"  >&lt;p&gt;Hi Dimitry,&lt;br/&gt;
&#160;We are using &lt;a href=&quot;https://downloads.percona.com/downloads/percona-server-mongodb-4.2/percona-server-mongodb-4.2.8-8/binary/debian/bionic/x86_64/percona-server-mongodb-4.2.8-8-r389dde5-bionic-x86_64-bundle.tar&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;percona-server-mongodb-4.2.8-8-r389dde5-bionic-x86_64-bundle.tar&lt;/a&gt;&#160;this binary.&#160;&lt;/p&gt;</comment>
                            <comment id="3569584" author="dmitry.agranat" created="Mon, 18 Jan 2021 13:29:23 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sourabh.ghosh%40visiblealpha.com&quot; class=&quot;user-hover&quot; rel=&quot;sourabh.ghosh@visiblealpha.com&quot;&gt;sourabh.ghosh@visiblealpha.com&lt;/a&gt;, thank you for providing the requested information. &lt;/p&gt;

&lt;p&gt;Could you please clarify what the 4.2.8-8 binary stands for?&lt;/p&gt;</comment>
                            <comment id="3564292" author="JIRAUSER1258112" created="Thu, 14 Jan 2021 06:17:40 +0000"  >&lt;p&gt;Thanks Dimitry,&lt;/p&gt;

&lt;p&gt;I have uploaded all the mongod and diagnostics logs on the above link as you mentioned&#160;&lt;br/&gt;
 below are the file and folder names as well kindly ignore any other files and folder&apos;s if you find.&lt;br/&gt;
 &lt;b&gt;Primary&lt;/b&gt; mongod files and folders&#160;&lt;b&gt;diagnostic.data_pri.gz/mongod.log_pri.gz&lt;/b&gt;&lt;br/&gt;
 &lt;b&gt;Secondary&lt;/b&gt; mongod files and folders&#160;&lt;b&gt;diagnostic.data_sec.gz/mongod.log_sec.gz&lt;/b&gt;&lt;br/&gt;
 &lt;b&gt;Secondary2&lt;/b&gt; mongod files and folders&#160;&lt;b&gt;diagnostic.data_sec2.gz/mongod.log_sec2.gz&lt;/b&gt;&lt;br/&gt;
 Let us know if anymore details you required from our end.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;</comment>
                            <comment id="3562271" author="dmitry.agranat" created="Wed, 13 Jan 2021 13:39:45 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sourabh.ghosh%40visiblealpha.com&quot; class=&quot;user-hover&quot; rel=&quot;sourabh.ghosh@visiblealpha.com&quot;&gt;sourabh.ghosh@visiblealpha.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Would you please archive (tar or zip) the mongod.log files and the &lt;tt&gt;$dbpath/diagnostic.data&lt;/tt&gt; directory (the contents are described &lt;a href=&quot;https://docs.mongodb.com/manual/administration/analyzing-mongodb-performance/#full-time-diagnostic-data-capture&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;here&lt;/a&gt;) from all members of the replica set and upload them to this &lt;a href=&quot;https://10gen-httpsupload.s3.amazonaws.com/upload_forms/2509004f-c61d-4f2c-aea0-8fda5e8e6081.html&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;support uploader&lt;/a&gt; location?&lt;/p&gt;

&lt;p&gt;Files uploaded to this portal are visible only to MongoDB employees and are routinely deleted after some time.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Dima&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="296228" name="Screen Shot 2021-01-20 at 17.18.41.png" size="362731" author="dmitry.agranat@mongodb.com" created="Wed, 20 Jan 2021 15:20:44 +0000"/>
                            <attachment id="296229" name="Screen Shot 2021-01-20 at 17.19.14.png" size="317632" author="dmitry.agranat@mongodb.com" created="Wed, 20 Jan 2021 15:20:56 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>11.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 13 Jan 2021 13:39:45 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        2 years, 51 weeks, 6 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>dmitry.agranat@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            2 years, 51 weeks, 6 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>dmitry.agranat@mongodb.com</customfieldvalue>
            <customfieldvalue>sourabh.ghosh@visiblealpha.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hypb4f:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hyb187:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[dmitry.agranat@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hyoxdr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>