<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:56:53 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-63095] Operations slow down during index build</title>
                <link>https://jira.mongodb.org/browse/SERVER-63095</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;The original problem: when we upgraded our MongoDB servers to 4.0.12, we started experiencing issues with building indexes. When building an index, the queries and update operations got hung during the time the index was built. To solve this problem, we would restart one of the secondary replicas and make that replica the primary one. That would completely resolve this issue.&lt;/p&gt;

&lt;p&gt;We were hoping that after upgrading our MongoDB servers to 4.2.17, the new optimized build process would completely eliminate the problem.&lt;/p&gt;

&lt;p&gt;However, when we tried to build an index with seven keys (not unique, with partial filter expression with two equality expressions) on a non-sharded collection with about 750k document (size - ~430 MB, storage size - ~170 MB), we observed the same behavior. The MongoDB server got very slow for the index build time. The queries got slowed down during the time the index was being built (that was about 66 seconds). The queries took several dozen seconds when usually they take less than 100 milliseconds.&lt;/p&gt;

&lt;p&gt;We tried to restart some of the secondaries and make it primary, and the index got built within several seconds and no slowdowns were observed.&lt;/p&gt;

&lt;p&gt;Our cluster configuration:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;shard cluster with 10 shards&lt;/li&gt;
	&lt;li&gt;four replicas in each shard&lt;/li&gt;
	&lt;li&gt;about 400 GB of data in storage size per shard&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Replica server configuration:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;Ubuntu 16.04&lt;/li&gt;
	&lt;li&gt;XSF&lt;/li&gt;
	&lt;li&gt;Kernel - 4.4.0-1128-aws #142-Ubuntu SMP Fri Apr 16 12:42:33 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux&lt;/li&gt;
	&lt;li&gt;Disable Transparent Huge&#160;disabled&lt;/li&gt;
	&lt;li&gt;AWS m5.2xlarge (8cpu\32gb)&lt;/li&gt;
	&lt;li&gt;SSD GP3 450 Gb&lt;/li&gt;
	&lt;li&gt;monogo-org-server - 4.2.17&lt;/li&gt;
&lt;/ul&gt;
</description>
                <environment>Ubuntu 16.04&lt;br/&gt;
XSF&lt;br/&gt;
Kernel - 4.4.0-1128-aws #142-Ubuntu SMP Fri Apr 16 12:42:33 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux&lt;br/&gt;
Disable Transparent Huge disabled&lt;br/&gt;
AWS m5.2xlarge (8cpu\32gb)&lt;br/&gt;
SSD GP3 450 Gb&lt;br/&gt;
monogo-org-server - 4.2.17</environment>
        <key id="1973213">SERVER-63095</key>
            <summary>Operations slow down during index build</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13202">Works as Designed</resolution>
                                        <assignee username="edwin.zhou@mongodb.com">Edwin Zhou</assignee>
                                    <reporter username="vladimirred456@gmail.com">Vladimir Beliakov</reporter>
                        <labels>
                            <label>index</label>
                            <label>slowdown,</label>
                    </labels>
                <created>Fri, 28 Jan 2022 16:38:23 +0000</created>
                <updated>Fri, 27 Oct 2023 13:52:07 +0000</updated>
                            <resolved>Wed, 17 Aug 2022 11:54:08 +0000</resolved>
                                    <version>4.2.17</version>
                                                                        <votes>0</votes>
                                    <watches>6</watches>
                                                                                                                <comments>
                            <comment id="4756980" author="JIRAUSER1257066" created="Wed, 17 Aug 2022 11:54:08 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=vladimirred456%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;vladimirred456@gmail.com&quot;&gt;vladimirred456@gmail.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;We&apos;re happy to hear that you haven&apos;t encountered a similar behavior with operations slowing down during an index build. Since our previous investigation has suggested the behavior you saw is expected, I&apos;ll close this ticket as Works as Designed.&lt;/p&gt;

&lt;p&gt;Best,&lt;br/&gt;
Edwin&lt;/p&gt;</comment>
                            <comment id="4545107" author="vladimirred456@gmail.com" created="Thu, 12 May 2022 14:54:54 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=edwin.zhou%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;edwin.zhou@mongodb.com&quot;&gt;edwin.zhou@mongodb.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Sorry for the late response. Unfortunately (or thankfully &lt;img class=&quot;emoticon&quot; src=&quot;https://jira.mongodb.org/images/icons/emoticons/smile.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt; ), we haven&apos;t been able to observe any similar behavior during any index builds. But if we do, I will attach the data you requested.&lt;/p&gt;</comment>
                            <comment id="4526188" author="JIRAUSER1257066" created="Wed, 4 May 2022 16:30:22 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=vladimirred456%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;vladimirred456@gmail.com&quot;&gt;vladimirred456@gmail.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;We still need additional information to diagnose the problem. If this is still an issue for you, would you please attach diagnostic data and logs that covers an index build that impacts other operations?&lt;/p&gt;</comment>
                            <comment id="4402038" author="JIRAUSER1257066" created="Wed, 9 Mar 2022 20:56:15 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=vladimirred456%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;vladimirred456@gmail.com&quot;&gt;vladimirred456@gmail.com&lt;/a&gt;,&lt;br/&gt;
 &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/364189/364189_Screen+Shot+2022-03-04+at+6.00.57+PM.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt; &lt;br/&gt;
Thank you for your patience while I investigate this incident. Based on the data that you provided about the most recent occurrence, we&apos;re not able to conclude that the behavior you observed is anything out of the ordinary. The increase in iowait doesn&apos;t tell us whether there is a regression.&lt;/p&gt;

&lt;p&gt;However, during the index build, we see disk utilization jump to 100% and is indicative of a disk bottleneck. Since an index build performs a collection scan, we can expect it to keep the disk busy while it scans data outside of the cache.&lt;/p&gt;

&lt;p&gt;Based on this incident, we can hypothesize that the previous incident, where other operations were impacted, may have been from a similar disk bottleneck, but we aren&apos;t able say for certain without data from the index build where operations are impacted.&lt;/p&gt;

&lt;p&gt;If you continue to see index builds impact other operations, please attach diagnostic data that covers those occurrences.&lt;/p&gt;

&lt;p&gt;Best,&lt;br/&gt;
Edwin&lt;/p&gt;</comment>
                            <comment id="4382645" author="vladimirred456@gmail.com" created="Tue, 1 Mar 2022 07:36:21 +0000"  >&lt;p&gt;Hi Edwin Zhou,&lt;/p&gt;

&lt;p&gt;It was 12:50 to 12:56 (UTC) Feb 17 2022.&lt;/p&gt;</comment>
                            <comment id="4380873" author="JIRAUSER1257066" created="Mon, 28 Feb 2022 16:51:32 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=vladimirred456%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;vladimirred456@gmail.com&quot;&gt;vladimirred456@gmail.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Thank you for following up with the diagnostic data for another occurrence. Can you please provide timestamps for when you performed the index build to help correlate the diagnostic data with the date and time of the incident and increased iowait? &lt;/p&gt;

&lt;p&gt;Best,&lt;br/&gt;
Edwin&lt;/p&gt;</comment>
                            <comment id="4367142" author="vladimirred456@gmail.com" created="Mon, 21 Feb 2022 18:23:59 +0000"  >&lt;p&gt;Hi, Edwin Zhou! Thanks for your reply. Unfortunately, we don&apos;t have `diagnostic.data` for the time when the incident happened. However, several days ago we encounter somewhat similar behavior when building indexes (but only with three keys, though). Even though the ongoing operations weren&apos;t inhibited, the primary replica and some secondary replica underwent an increase in iowait. I hope these attached files will shed some light on the culprit. And I&apos;ll keep in mind to collect&#160;`diagnostic.data`&#160; should we run into the same problem.&lt;br/&gt;
&lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/attachment/361689/361689_diagnostic.data.zip&quot; title=&quot;diagnostic.data.zip attached to SERVER-63095&quot;&gt;diagnostic.data.zip&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.mongodb.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt;&lt;/p&gt;</comment>
                            <comment id="4350759" author="JIRAUSER1257066" created="Fri, 11 Feb 2022 16:44:42 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=vladimirred456%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;vladimirred456@gmail.com&quot;&gt;vladimirred456@gmail.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Thank you for your report. Would you please archive (tar or zip) the &lt;tt&gt;$dbpath/diagnostic.data&lt;/tt&gt; directory (the contents are described &lt;a href=&quot;https://docs.mongodb.com/manual/administration/analyzing-mongodb-performance/#full-time-diagnostic-data-capture&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;here&lt;/a&gt;) that cover the incident you described and attach it to this ticket? &lt;/p&gt;

&lt;p&gt;Best,&lt;br/&gt;
Edwin&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                            <outwardlinks description="duplicates">
                                        <issuelink>
            <issuekey id="1973125">SERVER-63087</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="364189" name="Screen Shot 2022-03-04 at 6.00.57 PM.png" size="695573" author="edwin.zhou@mongodb.com" created="Fri, 4 Mar 2022 23:04:59 +0000"/>
                            <attachment id="361689" name="diagnostic.data.zip" size="62548757" author="vladimirred456@gmail.com" created="Mon, 21 Feb 2022 18:23:56 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>8.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 11 Feb 2022 16:44:42 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        1 year, 25 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            1 year, 25 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>edwin.zhou@mongodb.com</customfieldvalue>
            <customfieldvalue>vladimirred456@gmail.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i0ig5r:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|i01jiv:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10750" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Steps To Reproduce</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>&lt;p&gt;1. Use the configuration mentioned above.&lt;br/&gt;
2. Apply the following approximate amount of operations to the cluster:&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;query - 7k op/sec&lt;/li&gt;
	&lt;li&gt;command - 2.5k op/sec&lt;/li&gt;
	&lt;li&gt;getmore - 1k op/sec&lt;/li&gt;
	&lt;li&gt;insert - 500 op/sec&lt;/li&gt;
	&lt;li&gt;update - 300 op/sec&lt;/li&gt;
	&lt;li&gt;deletes - 100 op/sec&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;3. Create an index with seven key on a collection with about 750k document (size - ~430 MB, storage size - ~170 MB).&lt;/p&gt;</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[edwin.zhou@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i0i2b3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>