<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 06:09:57 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-68124] Primary replica member hangs during chunk migrating</title>
                <link>https://jira.mongodb.org/browse/SERVER-68124</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;We&apos;re using `sh.addTagRange` to set custom chunk ranges. And usually the ranges are fixed and the chunks are not moved between the shards. However, the other day we needed to change the ranges which made the balancer move chunks. Within 15-20 minutes afterwards the primary of some of the shards got unresponsive and the whole shard cluster hang.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/attachment/390794/390794_diagnostic.data.zip&quot; title=&quot;diagnostic.data.zip attached to SERVER-68124&quot;&gt;diagnostic.data.zip&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.mongodb.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt; of the hang primary is attached.&lt;/p&gt;


&lt;p&gt;Also,&#160;I found this message in the logs&lt;/p&gt;
&lt;blockquote&gt;&lt;p&gt;&lt;tt&gt;STORAGE &#160;&lt;span class=&quot;error&quot;&gt;&amp;#91;FlowControlRefresher&amp;#93;&lt;/span&gt;&#160;Flow control is engaged and the sustainer point is not moving. Please check the health of all secondaries.&lt;/tt&gt;&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;which led me to&#160;this issue. Might be related.&lt;/p&gt;</description>
                <environment>Ubuntu 16.04&lt;br/&gt;
XSF&lt;br/&gt;
Kernel - 4.4.0-1128-aws #142-Ubuntu SMP Fri Apr 16 12:42:33 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux&lt;br/&gt;
Disable Transparent Huge disabled&lt;br/&gt;
AWS m5.large (2cpu\8gb)&lt;br/&gt;
SSD GP3 450 Gb&lt;br/&gt;
monogo-org-server - 4.2.17</environment>
        <key id="2093882">SERVER-68124</key>
            <summary>Primary replica member hangs during chunk migrating</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="chris.kelly@mongodb.com">Chris Kelly</assignee>
                                    <reporter username="vladimirred456@gmail.com">Vladimir Beliakov</reporter>
                        <labels>
                            <label>balancing</label>
                    </labels>
                <created>Tue, 19 Jul 2022 07:26:25 +0000</created>
                <updated>Tue, 11 Oct 2022 03:09:18 +0000</updated>
                            <resolved>Tue, 11 Oct 2022 03:09:18 +0000</resolved>
                                    <version>4.2.17</version>
                                                                        <votes>0</votes>
                                    <watches>5</watches>
                                                                                                                <comments>
                            <comment id="4893002" author="JIRAUSER1265262" created="Tue, 11 Oct 2022 03:08:59 +0000"  >&lt;p&gt;We haven&#8217;t heard back from you for some time, so I&#8217;m going to close this ticket. If this is still an issue for you, please provide additional information and we will reopen the ticket.&lt;/p&gt;</comment>
                            <comment id="4857483" author="JIRAUSER1265262" created="Tue, 27 Sep 2022 07:10:46 +0000"  >&lt;p&gt;Hi Vladimir,&lt;/p&gt;

&lt;p&gt;Just a reminder, in order to investigate this further we need additional information. Specifically, we are missing:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;A specific timeline of what is happening on your end (with timestamps)&lt;/li&gt;
	&lt;li&gt;The mongod logs associated with the event (if they still exist)&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;We are missing indicators that relate your submitted diagnostic data to &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-53738&quot; title=&quot;Flow control is engaged and the sustainer point is not moving. Please check the health of all secondaries&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-53738&quot;&gt;&lt;del&gt;SERVER-53738&lt;/del&gt;&lt;/a&gt;. Specifically, moving chunks doesn&apos;t appear to happen in your diagnostic data at all in the time range of your diagnostic data (2022-02-23T00:30:04.584Z to 2022-02-23T00:30:04.584Z).&lt;/p&gt;

&lt;p&gt;If you have more data from the first incident, or the issue happens again, please submit the additional information so we can investigate further.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Regards,&lt;/p&gt;

&lt;p&gt;Christopher&lt;/p&gt;</comment>
                            <comment id="4813941" author="JIRAUSER1265262" created="Fri, 9 Sep 2022 09:43:37 +0000"  >&lt;p&gt;Vladimir,&lt;/p&gt;

&lt;p&gt;Thank you for your patience on this issue.&lt;/p&gt;

&lt;p&gt;If you still have the mongod logs for this issue (or are still experiencing the problem), can you please submit them in conjunction with the diagnostic data for more context?&lt;/p&gt;

&lt;p&gt;As for what I saw from your initial submission, I do not see the same indicators as are present in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-53738&quot; title=&quot;Flow control is engaged and the sustainer point is not moving. Please check the health of all secondaries&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-53738&quot;&gt;&lt;del&gt;SERVER-53738&lt;/del&gt;&lt;/a&gt;. Specifically, there is no recorded activity moving chunks on the primary or secondary here. FTDC is lost on your primary node at 2022-02-23T12:11:05.000Z.&#160;&lt;/p&gt;

&lt;p&gt;At point A (2022-02-23T12:10:11.053Z) there is a jump in the amount of insert ops recorded, which correlates to high disk utilization. In the middle of this, a checkpoint begins and we start to see the amount of active and queued writers rise quickly over the next few seconds.&lt;/p&gt;

&lt;p&gt;At point B (2022-02-23T12:10:40.291Z) we see a substantial number of application threads spent evicting (denoted by &lt;b&gt;ss wt thread-yield application thread time evicting&lt;/b&gt;). This can occur when wt cache dirty fill ratio exceeds 20%, which is also observed.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/400789/400789_image-2022-09-09-05-44-22-482.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;I&apos;m not immediately sure what to make of this and would be interested to get additional logs, and a more specific timeline of events on your side if at all possible. I&apos;d be interested in knowing when you&apos;re changing your chunk ranges, whether chunk migration is even occurring, or if this is something unrelated doing this. In the ticket you linked, we see indicators of moveChunk being initiated but I do not see that on either your primary or secondary data.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Regards,&lt;/p&gt;

&lt;p&gt;Christopher&#160;&lt;/p&gt;</comment>
                            <comment id="4691888" author="vladimirred456@gmail.com" created="Wed, 20 Jul 2022 14:06:29 +0000"  >&lt;blockquote&gt;
&lt;p&gt;which led me to&#160;this issue. Might be related.&lt;/p&gt;&lt;/blockquote&gt;


&lt;p&gt;Sorry, forgot to add the link. &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-53738&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;Here it is&lt;/a&gt;.&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="390794" name="diagnostic.data.zip" size="50721967" author="vladimirred456@gmail.com" created="Tue, 19 Jul 2022 07:24:01 +0000"/>
                            <attachment id="400789" name="image-2022-09-09-05-44-22-482.png" size="310040" author="chris.kelly@mongodb.com" created="Fri, 9 Sep 2022 09:44:22 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>4.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 20 Jul 2022 10:54:28 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        1 year, 17 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>chris.kelly@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            1 year, 17 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>chris.kelly@mongodb.com</customfieldvalue>
            <customfieldvalue>vladimirred456@gmail.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i12xfj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|i0lr6w:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[chris.kelly@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i12jkv:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>