<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 06:18:27 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-71246] Excessive CPU consumption of newly 5.0 upgraded instance </title>
                <link>https://jira.mongodb.org/browse/SERVER-71246</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;We&apos;re in the process of upgrading our MongoDB cluster from 4.4.15 to 5.0.10.&lt;br/&gt;
According to the upgrade documentation we stopped the balancer, then upgraded the config replica set.&lt;br/&gt;
Then we started upgrading the shards. After upgrading 3 out of 4 replicas we were ready to move primary onto already upgraded MongoDB instances.&lt;br/&gt;
Everything was ok for 9 of 10 shards.&lt;br/&gt;
However for the biggest shard (it contains most of the unsharded data) we ran into excessive CPU consumption. We tried to restart the mongod process, but to no avail.&lt;br/&gt;
Going over the logs didn&apos;t give us much insight regarding why the process was consuming so much CPU.&lt;br/&gt;
There were many &quot;Slow queries&quot; log records, which makes sense. Also we there were many &quot;WriteConflict&quot; log records like this one:&lt;/p&gt;

&lt;p&gt;{&quot;t&quot;:&lt;/p&gt;

{&quot;$date&quot;:&quot;2022-11-07T09:08:24.320+00:00&quot;}

&lt;p&gt;,&quot;s&quot;:&quot;W&quot;, &quot;c&quot;:&quot;COMMAND&quot;, &quot;id&quot;:23802, &quot;ctx&quot;:&quot;conn3798&quot;,&quot;msg&quot;:&quot;Plan executor error during findAndModify&quot;,&quot;attr&quot;:{&quot;error&quot;:&lt;/p&gt;

{&quot;code&quot;:112,&quot;codeName&quot;:&quot;WriteConflict&quot;,&quot;errmsg&quot;:&quot;WriteConflict error: this operation conflicted with another operation. Please retry your operation or multi-document transaction.&quot;}

&lt;p&gt;,&quot;stats&quot;:{&quot;stage&quot;:&quot;UPDATE&quot;,&quot;nReturned&quot;:0,&quot;executionTimeMillisEstimate&quot;:1,&quot;works&quot;:14,&quot;advanced&quot;:0,&quot;needTime&quot;:12,&quot;needYield&quot;:1,&quot;saveState&quot;:1,&quot;restoreState&quot;:1,&quot;failed&quot;:true,&quot;isEOF&quot;:1,&quot;nMatched&quot;:0,&quot;nWouldModify&quot;:0,&quot;nWouldUpsert&quot;:0,&quot;inputStage&quot;:{&quot;stage&quot;:&quot;LIMIT&quot;,&quot;nReturned&quot;:1,&quot;executionTimeMillisEstimate&quot;:1,&quot;works&quot;:13,&quot;advanced&quot;:1,&quot;needTime&quot;:12,&quot;needYield&quot;:0,&quot;saveState&quot;:2,&quot;restoreState&quot;:1,&quot;isEOF&quot;:1,&quot;limitAmount&quot;:1,&quot;inputStage&quot;:{&quot;stage&quot;:&quot;FETCH&quot;,&quot;nReturned&quot;:1,&quot;executionTimeMillisEstimate&quot;:1,&quot;works&quot;:13,&quot;advanced&quot;:1,&quot;needTime&quot;:12,&quot;needYield&quot;:0,&quot;saveState&quot;:2,&quot;restoreState&quot;:1,&quot;isEOF&quot;:0,&quot;docsExamined&quot;:1,&quot;alreadyHasObj&quot;:0,&quot;inputStage&quot;:{&quot;stage&quot;:&quot;SORT_MERGE&quot;,&quot;nReturned&quot;:1,&quot;executionTimeMillisEstimate&quot;:1,&quot;works&quot;:13,&quot;advanced&quot;:1,&quot;needTime&quot;:12,&quot;needYield&quot;:0,&quot;saveState&quot;:2,&quot;restoreState&quot;:1,&quot;isEOF&quot;:0,&quot;sortPattern&quot;:{&quot;&lt;/p&gt;

{REDACTED}&quot;:1,&quot;{REDACTED}&lt;br/&gt;
&lt;br/&gt;
&quot;:1},&quot;dupsTested&quot;:1,&quot;dupsDropped&quot;:0,&quot;inputStages&quot;:[{&quot;stage&quot;:&quot;IXSCAN&quot;,&quot;nReturned&quot;:1,&quot;executionTimeMillisEstimate&quot;:0,&quot;works&quot;:1,&quot;advanced&quot;:1,&quot;needTime&quot;:0,&quot;needYield&quot;:0,&quot;saveState&quot;:2,&quot;restoreState&quot;:1,&quot;isEOF&quot;:0,&quot;keyPattern&quot;:&lt;br/&gt;

{&quot;REDACTED&quot;}&lt;br/&gt;
&lt;br/&gt;
,&quot;indexName&quot;:&quot;&lt;br/&gt;
{REDACTED}
&lt;p&gt;&quot;,&quot;isMultiKey&quot;:false,&quot;multiKeyPaths&quot;:{&quot;&lt;/p&gt;
{REDACTED}&lt;br/&gt;
&lt;br/&gt;
&quot;:[],&quot;&lt;br/&gt;
{REDACTED}
&lt;p&gt;&quot;:[],&quot;{REDACTED}&lt;/p&gt;

&lt;p&gt;&quot;:[]},&quot;isUnique&quot;:false,&quot;isSparse&quot;:false,&quot;isPartial&quot;:false,&quot;indexVersion&quot;:2,&quot;direction&quot;:&quot;forward&quot;,&quot;indexBounds&quot;:{&quot;&lt;/p&gt;

{REDACTED}&quot;:[&quot;&lt;span class=&quot;error&quot;&gt;&amp;#91;0, 0&amp;#93;&lt;/span&gt;&quot;],&quot;{REDACTED}&lt;br/&gt;
&lt;br/&gt;
&quot;:[&quot;&lt;span class=&quot;error&quot;&gt;&amp;#91;MinKey, MaxKey&amp;#93;&lt;/span&gt;&quot;],&quot;&lt;br/&gt;
{REDACTED}
&lt;p&gt;&quot;:[&quot;&lt;span class=&quot;error&quot;&gt;&amp;#91;new Date(-9223372036854775808), new Date(1667812104306))&amp;quot;&amp;#93;&lt;/span&gt;},&quot;keysExamined&quot;:1,&quot;seeks&quot;:1,&quot;dupsTested&quot;:0,&quot;dupsDropped&quot;:0},{&quot;stage&quot;:&quot;IXSCAN&quot;,&quot;nReturned&quot;:0,&quot;executionTimeMillisEstimate&quot;:1,&quot;works&quot;:11,&quot;advanced&quot;:0,&quot;needTime&quot;:10,&quot;needYield&quot;:0,&quot;saveState&quot;:2,&quot;restoreState&quot;:1,&quot;isEOF&quot;:1,&quot;keyPattern&quot;:{&quot;&lt;/p&gt;
{REDACTED}&lt;br/&gt;
&lt;br/&gt;
&quot;:1,&quot;&lt;br/&gt;
{REDACTED}
&lt;p&gt;&quot;:1,&quot;{REDACTED}&lt;/p&gt;

&lt;p&gt;&quot;:1},&quot;indexName&quot;:&quot;&lt;/p&gt;

{REDACTED}&quot;,&quot;isMultiKey&quot;:false,&quot;multiKeyPaths&quot;:{&quot;{REDACTED}

&lt;p&gt;&quot;:[],&quot;&lt;/p&gt;

{REDACTED}&quot;:[],&quot;{REDACTED}&lt;br/&gt;
&lt;br/&gt;
&quot;:[]},&quot;isUnique&quot;:false,&quot;isSparse&quot;:false,&quot;isPartial&quot;:false,&quot;indexVersion&quot;:2,&quot;direction&quot;:&quot;forward&quot;,&quot;indexBounds&quot;:{&quot;&lt;br/&gt;
{REDACTED}
&lt;p&gt;&quot;:[&quot;&lt;span class=&quot;error&quot;&gt;&amp;#91;1, 1&amp;#93;&lt;/span&gt;&quot;],&quot;{REDACTED}&lt;/p&gt;

&lt;p&gt;&quot;:[&quot;&lt;span class=&quot;error&quot;&gt;&amp;#91;MinKey, MaxKey&amp;#93;&lt;/span&gt;&quot;],&quot;&lt;/p&gt;

{REDACTED}&quot;:[&quot;&lt;span class=&quot;error&quot;&gt;&amp;#91;new Date(-9223372036854775808), new Date(1667812104306))&amp;quot;&amp;#93;&lt;/span&gt;},&quot;keysExamined&quot;:11,&quot;seeks&quot;:11,&quot;dupsTested&quot;:0,&quot;dupsDropped&quot;:0}]}}}},&quot;cmd&quot;:{&quot;findAndModify&quot;:&quot;{REDACTED}

&lt;p&gt;&quot;,&quot;query&quot;:{&quot;&lt;/p&gt;

{REDACTED}&quot;:{&quot;$in&quot;:&lt;span class=&quot;error&quot;&gt;&amp;#91;0,1&amp;#93;&lt;/span&gt;},&quot;{REDACTED}&lt;br/&gt;
&lt;br/&gt;
&quot;:{&quot;$lt&quot;:&lt;br/&gt;

{&quot;$date&quot;:&quot;2022-11-07T09:08:24.306Z&quot;}&lt;br/&gt;
&lt;br/&gt;
}},&quot;sort&quot;:{&quot;&lt;br/&gt;
{REDACTED}
&lt;p&gt;&quot;:1,&quot;{REDACTED}&lt;/p&gt;

&lt;p&gt;&quot;:1},&quot;update&quot;:{&quot;$set&quot;:{&quot;&lt;/p&gt;

{REDACTED}&quot;:1,&quot;{REDACTED}&lt;br/&gt;
&lt;br/&gt;
&quot;:&lt;br/&gt;

{&quot;$date&quot;:&quot;2022-11-07T09:19:54.305Z&quot;}&lt;br/&gt;
&lt;br/&gt;
,&quot;&lt;br/&gt;
{REDACTED}
&lt;p&gt;&quot;:{&quot;$binary&quot;:{&quot;base64&quot;:&quot;hW7CcBc5Uki6JVK/hvU5ew==&quot;,&quot;subType&quot;:&quot;3&quot;}},&quot;{REDACTED}&lt;/p&gt;

&lt;p&gt;&quot;:{&quot;$date&quot;:&quot;2022-11-07T09:08:24.305Z&quot;}},&quot;$unset&quot;:{&quot;&lt;/p&gt;

{REDACTED}&quot;:1},&quot;$inc&quot;:{&quot;{REDACTED}

&lt;p&gt;&quot;:1}},&quot;runtimeConstants&quot;:{&quot;localNow&quot;:&lt;/p&gt;

{&quot;$date&quot;:&quot;2022-11-07T09:08:24.305Z&quot;}

&lt;p&gt;,&quot;clusterTime&quot;:{&quot;$timestamp&quot;:&lt;/p&gt;

{&quot;t&quot;:1667812104,&quot;i&quot;:2118}

&lt;p&gt;}},&quot;writeConcern&quot;:{&quot;w&quot;:1,&quot;wtimeout&quot;:0,&quot;provenance&quot;:&quot;implicitDefault&quot;}}}}&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;We&apos;re urgently need help, because this is a show stopper for our MongoDB cluster optimization.&lt;/p&gt;

&lt;p&gt;Please let me know if any additional information is required.&lt;/p&gt;</description>
                <environment>Ubuntu 18.04.6 LTS&lt;br/&gt;
XSF&lt;br/&gt;
Kernel - 5.4.0-1088-aws #96~18.04.1-Ubuntu SMP Mon Oct 17 02:57:48 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux&lt;br/&gt;
Disable Transparent Huge disabled&lt;br/&gt;
AWS m5.2xlarge&lt;br/&gt;
SSD GP3 450 Gb</environment>
        <key id="2180845">SERVER-71246</key>
            <summary>Excessive CPU consumption of newly 5.0 upgraded instance </summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="yuan.fang@mongodb.com">Yuan Fang</assignee>
                                    <reporter username="vladimirred456@gmail.com">Vladimir Beliakov</reporter>
                        <labels>
                    </labels>
                <created>Thu, 10 Nov 2022 15:59:30 +0000</created>
                <updated>Fri, 11 Nov 2022 05:46:21 +0000</updated>
                            <resolved>Thu, 10 Nov 2022 22:03:02 +0000</resolved>
                                                                                        <votes>0</votes>
                                    <watches>6</watches>
                                                                                                                <comments>
                            <comment id="4973244" author="vladimirred456@gmail.com" created="Fri, 11 Nov 2022 05:46:21 +0000"  >&lt;p&gt;Hi Yuan Fang,&lt;/p&gt;

&lt;p&gt;Thank you for looking into this!&lt;br/&gt;
The spike of updates you mentioned is probably related to the instance becoming primary. And since that instance is the biggest shard which holds a lot of data and usually receives a lot writes, I assume the update spike is related to that.&lt;br/&gt;
Surprisingly, we don&apos;t see the same behavior on the other 9 shards. Usually they receive less writes, but no CPU consumption increase was observed by us.&lt;/p&gt;</comment>
                            <comment id="4972522" author="JIRAUSER1270794" created="Thu, 10 Nov 2022 21:58:10 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=vladimirred456%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;vladimirred456@gmail.com&quot;&gt;vladimirred456@gmail.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Thank you for your report. With regards to your question:&lt;/p&gt;
&lt;blockquote&gt;&lt;p&gt;Going over the logs didn&apos;t give us much insight regarding why the process was consuming so much CPU.&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/414411/414411_Screen+Shot+2022-11-10+at+4.46.12+PM.png&quot; height=&quot;402&quot; width=&quot;586&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;By looking at the diagnostic data you provided. At point A (the point upgrade to 5.0) through point B, I can see spikes of update, which seem correlated to write conflicts spikes. The write conflicts and update went down to 0 after point B, meanwhile, the CPU utilization also went down. Therefore, the high CPU utilization after the upgrade is likely caused by an increase in update operations.&lt;/p&gt;

&lt;p&gt;As for what causes the occurrence of write conflicts, I don&apos;t have a clear answer, usually, it is due to multiple clients trying to update one document at the same time. I would check the oplog to find more information if write conflicts occur again.&lt;/p&gt;

&lt;p&gt;For this issue, we&apos;d like to encourage you to start by asking our community for help by posting on the &lt;a href=&quot;https://www.mongodb.com/community/forums&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;MongoDB Developer Community Forums&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;If the discussion there leads you to suspect a bug in the MongoDB server, then we&apos;d want to investigate it as a possible bug here in the SERVER project.&lt;br/&gt;
&#160;&lt;br/&gt;
Regards,&lt;br/&gt;
Yuan&lt;/p&gt;</comment>
                            <comment id="4972208" author="vladimirred456@gmail.com" created="Thu, 10 Nov 2022 20:39:59 +0000"  >&lt;p&gt;Sorry, I forgot to mention that the incident (when we switched the primary to the 5.0 upgraded mongod instance) was happening from 2022-11-10 09:03:00 to 2022-11-10 09:07:00.&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="414411" name="Screen Shot 2022-11-10 at 4.46.12 PM.png" size="276559" author="yuan.fang@mongodb.com" created="Thu, 10 Nov 2022 21:58:51 +0000"/>
                            <attachment id="414298" name="diagnostics.data.zip" size="27214281" author="vladimirred456@gmail.com" created="Thu, 10 Nov 2022 16:00:26 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>3.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Thu, 10 Nov 2022 21:58:10 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        1 year, 12 weeks, 6 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>vladimirred456@gmail.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            1 year, 12 weeks, 6 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>vladimirred456@gmail.com</customfieldvalue>
            <customfieldvalue>yuan.fang@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i1hsfb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|i10eqo:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[yuan.fang@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i1hekn:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>