<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:15:32 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-7765] Draining a shard stalled due to writebacksQueued stalled</title>
                <link>https://jira.mongodb.org/browse/SERVER-7765</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;I started draining the last of four shards in a live sharded mongo cluster (v2.0.7), with each shard being a 3-node replset, and it went fine until it got to 16 chunks remaining. Now the draining has been stuck there for more than four hours.&lt;/p&gt;

&lt;p&gt;mongos&amp;gt; db.runCommand(&lt;/p&gt;
{removeShard:&quot;mongo-live-d&quot;}
&lt;p&gt;)&lt;br/&gt;
{&lt;br/&gt;
        &quot;msg&quot; : &quot;draining ongoing&quot;,&lt;br/&gt;
        &quot;state&quot; : &quot;ongoing&quot;,&lt;br/&gt;
        &quot;remaining&quot; : &lt;/p&gt;
{
                &quot;chunks&quot; : NumberLong(16),
                &quot;dbs&quot; : NumberLong(0)
        }
&lt;p&gt;,&lt;br/&gt;
        &quot;ok&quot; : 1&lt;br/&gt;
}&lt;/p&gt;

&lt;p&gt;The mongos log shows this:&lt;/p&gt;

&lt;p&gt;Wed Nov 21 22:10:26 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; distributed lock &apos;balancer/mongo-live-a-1:27017:1350073653:1804289383&apos; acquired, ts : 50adc1d2538fcedc6aa3cf93&lt;br/&gt;
Wed Nov 21 22:10:26 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; biggest shard mongo-live-b has unprocessed writebacks, waiting for completion of migrate&lt;br/&gt;
Wed Nov 21 22:10:26 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; biggest shard mongo-live-b has unprocessed writebacks, waiting for completion of migrate&lt;br/&gt;
Wed Nov 21 22:10:26 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; biggest shard mongo-live-b has unprocessed writebacks, waiting for completion of migrate&lt;br/&gt;
Wed Nov 21 22:10:26 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; distributed lock &apos;balancer/mongo-live-a-1:27017:1350073653:1804289383&apos; unlocked. &lt;/p&gt;

&lt;p&gt;When I check writebacksQueued the total ops never goes down but is increasing over time:&lt;/p&gt;

&lt;p&gt;PRIMARY&amp;gt; db.adminCommand(&quot;writeBacksQueued&quot;)&lt;br/&gt;
{&lt;br/&gt;
        &quot;hasOpsQueued&quot; : true,&lt;br/&gt;
        &quot;totalOpsQueued&quot; : 603910,&lt;br/&gt;
        &quot;queues&quot; : {                &quot;50787cba376f032868ac165e&quot; : &lt;/p&gt;
{                        &quot;n&quot; : 0,
                        &quot;minutesSinceLastCall&quot; : 2
                }
&lt;p&gt;,&lt;br/&gt;
                &quot;50787cba4a4a812e093429a5&quot; : &lt;/p&gt;
{
                        &quot;n&quot; : 341466,
                        &quot;minutesSinceLastCall&quot; : 0
                }
&lt;p&gt;,&lt;br/&gt;
                &quot;50787cba5df1e05fedab56ff&quot; : &lt;/p&gt;
{
                        &quot;n&quot; : 1,
                        &quot;minutesSinceLastCall&quot; : 40
                }
&lt;p&gt;,&lt;br/&gt;
                &quot;50787cbadc8a4a2ee5bab98f&quot; : &lt;/p&gt;
{
                        &quot;n&quot; : 262443,
                        &quot;minutesSinceLastCall&quot; : 0
                }
&lt;p&gt;,&lt;br/&gt;
                &quot;50787cbafb83be34cb49a885&quot; : &lt;/p&gt;
{
                        &quot;n&quot; : 0,
                        &quot;minutesSinceLastCall&quot; : 1
                }
&lt;p&gt;        },&lt;br/&gt;
        &quot;ok&quot; : 1&lt;br/&gt;
}&lt;/p&gt;

&lt;p&gt;The &quot;totalOpsQueued&quot; and various &quot;n&quot; values keep going up. I don&apos;t see anything interesting in the troublesome shard&apos;s mongod log. I&apos;d try restarting everything but I&apos;m worried that this queued data would be lost.&lt;/p&gt;</description>
                <environment></environment>
        <key id="57528">SERVER-7765</key>
            <summary>Draining a shard stalled due to writebacksQueued stalled</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="4">Incomplete</resolution>
                                        <assignee username="barrie">Barrie Segal</assignee>
                                    <reporter username="papercrane">Justin Patrin</reporter>
                        <labels>
                    </labels>
                <created>Mon, 26 Nov 2012 21:21:10 +0000</created>
                <updated>Fri, 8 Mar 2013 15:56:10 +0000</updated>
                            <resolved>Tue, 19 Feb 2013 19:02:16 +0000</resolved>
                                    <version>2.0.7</version>
                                                                        <votes>0</votes>
                                    <watches>6</watches>
                                                                                                                <comments>
                            <comment id="261355" author="barrie" created="Fri, 8 Feb 2013 21:36:55 +0000"  >&lt;p&gt;Justin,&lt;/p&gt;

&lt;p&gt;Just checking in-- what was the outcome of draining the shard?&lt;/p&gt;

&lt;p&gt;Barrie &lt;/p&gt;</comment>
                            <comment id="202793" author="papercrane" created="Fri, 30 Nov 2012 02:43:37 +0000"  >&lt;p&gt;Well, we tried restarting the mongod processes on the secondaries of the misbehaving shard/repl. No change. We stepped down the primary. No change. Stopping the former primary once it was a secondary, however, took quite a while (~5 minutes). Once that was down we started it back up and the cluster is draining the shard like I originally had it doing again. I hope we haven&apos;t lost any data....&lt;/p&gt;</comment>
                            <comment id="200386" author="papercrane" created="Tue, 27 Nov 2012 18:46:38 +0000"  >&lt;p&gt;All of the mongos logs have the same Balancer entries as listed above, nothing different.&lt;/p&gt;

&lt;p&gt;Looking at the mongod logs I&apos;m seeing a fair number of entries like this:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Tue Nov 27 10:44:43 [conn3062] command admin.$cmd command: { writebacklisten: ObjectId(&apos;50787cbae13db536af5c54a0&apos;) } ntoreturn:1 reslen:44 300008ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="199748" author="eliot" created="Tue, 27 Nov 2012 03:46:17 +0000"  >&lt;p&gt;Can you check the mongos logs?&lt;br/&gt;
One of them should be processing a lot of write backs or have a lot of errors.&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>4.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 27 Nov 2012 03:46:17 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        11 years, 1 week, 5 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ian@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            11 years, 1 week, 5 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10000" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Old_Backport</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10000"><![CDATA[No]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10020"><![CDATA[Linux]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>barrie</customfieldvalue>
            <customfieldvalue>eliot</customfieldvalue>
            <customfieldvalue>papercrane</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrnghb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrkrsn:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>34962</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hsftjr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>