<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 06:10:17 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-68245] Degraded performance after 4.4 version upgrade</title>
                <link>https://jira.mongodb.org/browse/SERVER-68245</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Hi,&lt;/p&gt;

&lt;p&gt;We &lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/attachment/391633/391633_metrics.2022-07-22T00-20-13Z-00000&quot; title=&quot;metrics.2022-07-22T00-20-13Z-00000 attached to SERVER-68245&quot;&gt;metrics.2022-07-22T00-20-13Z-00000&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.mongodb.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt; replicaset, with 3 nodes. We were earlier on version 4.2&lt;br/&gt;
Average Response time: 1.5 ms&lt;br/&gt;
p55: 5ms&lt;/p&gt;

&lt;p&gt;The version upgrade to 4.4 was done around 21st July 10:00 pm. Immediately after the upgrade we did not notice any issues, but after a few hours, we started&#160;&lt;/p&gt;

&lt;p&gt;seeing an increase in response time and a lot of slow queries on mongo side.&lt;br/&gt;
Average Response time: 3.5 ms&lt;br/&gt;
p55: 15ms&lt;/p&gt;

&lt;p&gt;Data size: 5TB(uncompressed)&lt;br/&gt;
Index size: 35GB&lt;/p&gt;

&lt;p&gt;We also observed the following:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;Increase in memory page faults&lt;/li&gt;
	&lt;li&gt;Dirty data and dirty pages in cache&lt;/li&gt;
	&lt;li&gt;Increase in blocks read&lt;/li&gt;
	&lt;li&gt;Decrease in blocks written&lt;/li&gt;
	&lt;li&gt;Increase in cache evictions&lt;/li&gt;
	&lt;li&gt;Increase in cache pages read&lt;/li&gt;
	&lt;li&gt;Decrease in cache pages written&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Please find attached the screenshots and the diagnostic data.&lt;/p&gt;

&lt;p&gt;Please help us in finding out the issue and suggest an appropriate fix for this.&lt;br/&gt;
Thanks in advance!&lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/attachment/391634/391634_metrics.2022-07-22T05-45-13Z-00000&quot; title=&quot;metrics.2022-07-22T05-45-13Z-00000 attached to SERVER-68245&quot;&gt;metrics.2022-07-22T05-45-13Z-00000&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.mongodb.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/391635/391635_cache+pages+read+and+written+to+disk.png&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;&lt;img src=&quot;https://jira.mongodb.org/images/icons/attach/noimage.png&quot; imagetext=&quot;cache pages read:written to disk.png&quot; align=&quot;absmiddle&quot; border=&quot;0&quot; /&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/391640/391640_cahce+eviction.png&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/391639/391639_blocks+read+vs+vblocks+written.png&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/391638/391638_blocks+read.png&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/391637/391637_dirty+cache.png&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;</description>
                <environment></environment>
        <key id="2097571">SERVER-68245</key>
            <summary>Degraded performance after 4.4 version upgrade</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="chris.kelly@mongodb.com">Chris Kelly</assignee>
                                    <reporter username="ishrat.jahan@gojek.com">Ishrat Jahan</reporter>
                        <labels>
                            <label>performance</label>
                    </labels>
                <created>Sat, 23 Jul 2022 11:27:16 +0000</created>
                <updated>Tue, 18 Oct 2022 20:01:44 +0000</updated>
                            <resolved>Tue, 18 Oct 2022 20:01:44 +0000</resolved>
                                    <version>4.4.15</version>
                                                                        <votes>0</votes>
                                    <watches>8</watches>
                                                                                                                <comments>
                            <comment id="4910903" author="JIRAUSER1265262" created="Tue, 18 Oct 2022 20:01:34 +0000"  >&lt;p&gt;We haven&#8217;t heard back from you for some time, so I&#8217;m going to close this ticket. If this is still an issue for you, please provide additional information and we will reopen the ticket.&lt;/p&gt;</comment>
                            <comment id="4877369" author="JIRAUSER1265262" created="Tue, 4 Oct 2022 17:38:22 +0000"  >&lt;p&gt;Tejas,&lt;/p&gt;

&lt;p&gt;We still need additional information to diagnose the problem. If this is still an issue for you, would you please supply the requested information? Specifically, it is difficult to ascertain what degradation you&apos;re seeing without an example:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;A workload that is known to be identical, such as a test workload that can be repeated&lt;/li&gt;
	&lt;li&gt;Diagnostic data and mongodb logs for comparative tests of that repeatable workload on both 4.2 and 4.4&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Christopher&lt;/p&gt;</comment>
                            <comment id="4820280" author="JIRAUSER1265262" created="Mon, 12 Sep 2022 19:28:19 +0000"  >&lt;p&gt;Hi Tejas,&lt;/p&gt;

&lt;p&gt;Thank you for your patience on this. In order to investigate this further, it is imperative we get something that is reproducible. If you are unable to downgrade to 4.2 to collect diagnostic data for comparison, it is at least necessary to &lt;b&gt;provide a workload that is known to be cause this problem, such as a test workload that can be repeated on 4.2 and 4.4.&lt;/b&gt;&#160;&lt;/p&gt;

&lt;p&gt;As Eric mentioned, because of the differences in 4.2 and 4.4, asking our community for help by posting on the &lt;a href=&quot;https://www.mongodb.com/community/forums&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;MongoDB Developer Community Forums&lt;/a&gt; may prove helpful for tuning your performance.&lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
Christopher&lt;/p&gt;</comment>
                            <comment id="4759804" author="JIRAUSER1269677" created="Thu, 18 Aug 2022 10:41:31 +0000"  >&lt;p&gt;Hello Eric,&lt;/p&gt;

&lt;p&gt;We are not able to repeat this on other workload since this is production instance and it would be hard to replicate the traffic as well as the setup. Nonetheless, we are still seeing the response time spikes everyday. We also won&apos;t be able to provide any diagnostic data for 4.2 since that would require a downgrade. Is there anything else we can provide on the current setup?&lt;/p&gt;

&lt;p&gt;Also, all times in the above screenshots are in IST (+05:30).&lt;/p&gt;</comment>
                            <comment id="4746948" author="eric.sedor" created="Fri, 12 Aug 2022 15:06:53 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=ishrat.jahan%40gojek.com&quot; class=&quot;user-hover&quot; rel=&quot;ishrat.jahan@gojek.com&quot;&gt;ishrat.jahan@gojek.com&lt;/a&gt;, and apologies for the delay addressing this report. I want to start by setting some expectations:&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;There are some underlying storage engine changes between 4.2 and 4.4 that could be impacting this workload, but the magnitude of of the changes will make a direct comparison difficult&lt;/li&gt;
	&lt;li&gt;In looking at the provided diagnostic data, there are a few concerns I have:
	&lt;ul&gt;
		&lt;li&gt;The time period of this data is ~2022-07-22T00:20:14.000Z to ~2022-07-22T07:54:00.008Z - If the timestamps in your provided screenshots are in UTC, that seems to coincide with the lowest load period&lt;/li&gt;
		&lt;li&gt;Within this range, the number of active writers and average latency maximums of ~20ms are very low in real terms. Such small ranges of performance variation can be very difficult to attribute given the 1 second sample rate of diagnostic data&lt;/li&gt;
	&lt;/ul&gt;
	&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;To have the best chance of implicating a cause of a performance regression like this, we would really need to see:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;A workload that is known to be identical, such as a test workload that can be repeated&lt;/li&gt;
	&lt;li&gt;Diagnostic data and mongodb logs for comparative tests of that repeatable workload on both 4.2 and 4.4&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Are you able to provide this?&lt;/p&gt;

&lt;p&gt;Finally, and because of the differences in 4.2 and 4.4, I might also suggest treating this as a performance tuning effort on 4.4 and asking our community for help by posting on the &lt;a href=&quot;https://www.mongodb.com/community/forums&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;MongoDB Developer Community Forums&lt;/a&gt;. They might be able to help zero in on specific improvements to make.&lt;/p&gt;

&lt;p&gt;Sincerely,&lt;br/&gt;
Eric&lt;/p&gt;</comment>
                            <comment id="4717211" author="JIRAUSER1265575" created="Mon, 1 Aug 2022 07:02:17 +0000"  >&lt;p&gt;Our p99 response time has spiked till 22ms. Can someone please help check this ?&#160;&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="391636" name="blocks read vs vblocks written-1.png" size="99316" author="ishrat.jahan@gojek.com" created="Sat, 23 Jul 2022 11:21:07 +0000"/>
                            <attachment id="391639" name="blocks read vs vblocks written.png" size="99316" author="ishrat.jahan@gojek.com" created="Sat, 23 Jul 2022 11:20:23 +0000"/>
                            <attachment id="391638" name="blocks read.png" size="91435" author="ishrat.jahan@gojek.com" created="Sat, 23 Jul 2022 11:20:23 +0000"/>
                            <attachment id="391635" name="cache pages read and written to disk.png" size="101819" author="ishrat.jahan@gojek.com" created="Sat, 23 Jul 2022 11:24:01 +0000"/>
                            <attachment id="391640" name="cahce eviction.png" size="202177" author="ishrat.jahan@gojek.com" created="Sat, 23 Jul 2022 11:20:23 +0000"/>
                            <attachment id="391637" name="dirty cache.png" size="207586" author="ishrat.jahan@gojek.com" created="Sat, 23 Jul 2022 11:20:23 +0000"/>
                            <attachment id="391633" name="metrics.2022-07-22T00-20-13Z-00000" size="10455168" author="ishrat.jahan@gojek.com" created="Sat, 23 Jul 2022 11:27:14 +0000"/>
                            <attachment id="391634" name="metrics.2022-07-22T05-45-13Z-00000" size="4372222" author="ishrat.jahan@gojek.com" created="Sat, 23 Jul 2022 11:26:48 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>6.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 12 Aug 2022 15:06:53 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        1 year, 16 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>chris.kelly@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            1 year, 16 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>chris.kelly@mongodb.com</customfieldvalue>
            <customfieldvalue>eric.sedor@mongodb.com</customfieldvalue>
            <customfieldvalue>ishrat.jahan@gojek.com</customfieldvalue>
            <customfieldvalue>tejas.jadhav@gojek.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i13k6f:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|i0mdq8:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[eric.sedor@mongodb.com]]></customfieldvalue>
        <customfieldvalue><![CDATA[chris.kelly@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i136br:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>