<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:10:07 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-25760] Mongodump taking extraordinarily long, utilizing almost zero resources, yet slowing down server</title>
                <link>https://jira.mongodb.org/browse/SERVER-25760</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;When performing a basic mongodump on our database, with settings seen below, on high performance hardware, we&apos;re seeing very long delays between writes, and nearly zero resources on the system being utilized, but significantly increased response time to even the most simple of commands.&lt;/p&gt;

&lt;p&gt;CPU utilization remains the same as other systems. iostat shows nearly no activity, similar to other systems. Network load remains the same. &lt;/p&gt;

&lt;p&gt;The time it takes across NFS is just as long, and more easily shows the prolonged gaps in activity. You will see random peaks of full bandwidth usage lasting a few seconds, followed by several seconds, sometimes minutes of zero activity.&lt;/p&gt;
</description>
                <environment></environment>
        <key id="311311">SERVER-25760</key>
            <summary>Mongodump taking extraordinarily long, utilizing almost zero resources, yet slowing down server</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="3">Duplicate</resolution>
                                        <assignee username="kelsey.schubert@mongodb.com">Kelsey Schubert</assignee>
                                    <reporter username="sallgeud">Chad Kreimendahl</reporter>
                        <labels>
                    </labels>
                <created>Tue, 23 Aug 2016 19:22:37 +0000</created>
                <updated>Wed, 1 Feb 2017 00:45:34 +0000</updated>
                            <resolved>Wed, 1 Feb 2017 00:40:51 +0000</resolved>
                                    <version>3.2.7</version>
                                                    <component>Performance</component>
                    <component>Tools</component>
                                        <votes>1</votes>
                                    <watches>8</watches>
                                                                                                                <comments>
                            <comment id="1489632" author="thomas.schubert" created="Wed, 1 Feb 2017 00:40:24 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sallgeud&quot; class=&quot;user-hover&quot; rel=&quot;sallgeud&quot;&gt;sallgeud&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Thanks for confirming you haven&apos;t encountered &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-25070&quot; title=&quot;Significant delays when large collections are responsible for full WiredTiger cache&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-25070&quot;&gt;&lt;del&gt;SERVER-25070&lt;/del&gt;&lt;/a&gt; since upgrading to MongoDB 3.2.10. My understanding is that this issue has been resolved as well, so I&apos;m updating the ticket. Please let me know if that isn&apos;t the case.&lt;/p&gt;

&lt;p&gt;Thank you,&lt;br/&gt;
Thomas&lt;/p&gt;</comment>
                            <comment id="1399392" author="thomas.schubert" created="Tue, 4 Oct 2016 00:11:38 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sallgeud&quot; class=&quot;user-hover&quot; rel=&quot;sallgeud&quot;&gt;sallgeud&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Thanks for the additional information. I&apos;m going to put this ticket on hold, pending our investigation of &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-25070&quot; title=&quot;Significant delays when large collections are responsible for full WiredTiger cache&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-25070&quot;&gt;&lt;del&gt;SERVER-25070&lt;/del&gt;&lt;/a&gt; since it is likely the root cause. Please note that MongoDB 3.2.10 is available and ready for download. &lt;/p&gt;

&lt;p&gt;Let us know if it resolves the issues you have been observing.&lt;/p&gt;

&lt;p&gt;Thank you,&lt;br/&gt;
Thomas&lt;/p&gt;</comment>
                            <comment id="1389898" author="sallgeud" created="Wed, 21 Sep 2016 16:04:12 +0000"  >&lt;p&gt;1. We have a monitoring tool that outputs it all to a pretty dashboard for us.&lt;br/&gt;
2. I&apos;ve not yet tried this, but can.&lt;br/&gt;
3. We actually already do mongodump from two places, so that we have not only redundant data centers, but redundant recovery in each. We see the same issue in both. One is a regular secondary, the other is a hidden secondary.  The hidden has half the memory, and is the one that runs into trouble more often.  Since updating the hidden one to 3.2.9, we haven&apos;t seen it crash. However, we also went away from &quot;-j 8&quot; and just left it as a regular backup with no extra arguments.&lt;br/&gt;
4. For this specific database: Approximately 44 million objects across 92 databases. A total of about 90GB uncompressed data + indexes.&lt;br/&gt;
    a) We typically try to keep the databases sized so that we can fit the entirety of customer data + indexes in memory. So depending on the database, that&apos;s between 100GB and 500GB of data.&lt;/p&gt;</comment>
                            <comment id="1377481" author="thomas.schubert" created="Tue, 6 Sep 2016 17:47:34 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sallgeud&quot; class=&quot;user-hover&quot; rel=&quot;sallgeud&quot;&gt;sallgeud&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Thank you for the information, I have some follow up questions to help us continue to investigate:&lt;/p&gt;

&lt;ol&gt;
	&lt;li&gt;How are you measuring memory usage?&lt;/li&gt;
	&lt;li&gt;Do you observe the same issue if you dump a single database?&lt;/li&gt;
	&lt;li&gt;Are you able to execute &lt;tt&gt;mongodump&lt;/tt&gt; against the secondary from a different machine?&lt;/li&gt;
	&lt;li&gt;Do you have a rough estimate of total number of documents, and total size of your documents?&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;Thanks again,&lt;br/&gt;
Thomas&lt;/p&gt;</comment>
                            <comment id="1370070" author="sallgeud" created="Sun, 28 Aug 2016 15:38:40 +0000"  >&lt;p&gt;I can use mongodump to create the scenario you see in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-25663&quot; title=&quot;Odd connection timeouts and rejections when replicaset secondary is lagged&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-25663&quot;&gt;&lt;del&gt;SERVER-25663&lt;/del&gt;&lt;/a&gt; . &lt;/p&gt;

&lt;p&gt;Of note is that the system on which mongodump is running goes from using a regular amount of memory (~55% of the total &lt;span class=&quot;error&quot;&gt;&amp;#91;64GB&amp;#93;&lt;/span&gt;), and though it continues to report only 55% usage, and no other processes are running, you end up with &amp;gt; 100% of physical memory needing to be allocated, resulting in swap usage. &lt;/p&gt;

&lt;p&gt;I&apos;m curious if this may be the underlying issue. Ultimately we&apos;re swapping on ultra-fast nvme ssd, but I&apos;m not sure how exactly we got to swap.&lt;/p&gt;

&lt;p&gt;I&apos;ve got a great chart from our monitoring tools that shows it happening on the first backup post-restart. &lt;/p&gt;</comment>
                            <comment id="1369537" author="thomas.schubert" created="Fri, 26 Aug 2016 21:04:26 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sallgeud&quot; class=&quot;user-hover&quot; rel=&quot;sallgeud&quot;&gt;sallgeud&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Would you please clarify whether this is the same issue that we are investigating in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-25663&quot; title=&quot;Odd connection timeouts and rejections when replicaset secondary is lagged&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-25663&quot;&gt;&lt;del&gt;SERVER-25663&lt;/del&gt;&lt;/a&gt;?&lt;/p&gt;

&lt;p&gt;Thank you,&lt;br/&gt;
Thomas&lt;/p&gt;</comment>
                            <comment id="1365450" author="sallgeud" created="Tue, 23 Aug 2016 19:23:30 +0000"  >&lt;p&gt;I forgot to mention in my description:&lt;/p&gt;

&lt;p&gt;While the backup is running, simple commands such as &quot;show dbs&quot; take 10-40 times longer to respond.&lt;/p&gt;

&lt;p&gt;Regardless of the threading setting, the same results are seen. -j 1, -j 4, -j 8, -j 16, -j 24 all tested. &lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                            <outwardlinks description="duplicates">
                                        <issuelink>
            <issuekey id="301191">SERVER-25070</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="136739" name="MongoDumpCausingMongodMemoryConsumption.png" size="9479" author="sallgeud" created="Sun, 28 Aug 2016 15:39:23 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>7.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 26 Aug 2016 21:04:26 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        7 years, 2 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>kelsey.schubert@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            7 years, 2 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>sallgeud</customfieldvalue>
            <customfieldvalue>kelsey.schubert@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrjy4v:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hsp0tj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10750" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Steps To Reproduce</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>&lt;p&gt;MongoDB in 3x replicaset, with standard configuration.&lt;/p&gt;

&lt;p&gt;From the 3rd system (SECONDARY):&lt;br/&gt;
   mongodump -v -j 24 -o /backups/mongoRepl1&lt;/p&gt;</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[kelsey.schubert@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrlgnr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>