<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:26:07 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-31141] Poor performance with 95% cache and many writers</title>
                <link>https://jira.mongodb.org/browse/SERVER-31141</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;On mongo 3.4.9 we are seeing poor performance when the cache hits 95% utilization. I&apos;ve attached a snippet of mongostat output from a time when this was happening. When this slowdown happens there is almost no disk activity.&lt;/p&gt;

&lt;p&gt;Some additional information about our environment:&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;The server is running on an r4.8xlarge instance (244GB RAM / 32 cores)&lt;/li&gt;
	&lt;li&gt;We are using gzip compression&lt;/li&gt;
	&lt;li&gt;We have many concurrent writers and when this slow down happens we max out all of the write tickets. (Is it possible that eviction is having trouble making progress with that many application threads?)&lt;/li&gt;
	&lt;li&gt;This collection has 12 indexes.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Are there any configuration settings we should try changing to work around this?&lt;/p&gt;

&lt;p&gt;I can upload the diagnostic.data to the private portal if that would help.&lt;/p&gt;</description>
                <environment></environment>
        <key id="429001">SERVER-31141</key>
            <summary>Poor performance with 95% cache and many writers</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="alexander.gorrod@mongodb.com">Alexander Gorrod</assignee>
                                    <reporter username="bpot">Bob Potter</reporter>
                        <labels>
                            <label>MAREF</label>
                    </labels>
                <created>Mon, 18 Sep 2017 20:26:18 +0000</created>
                <updated>Mon, 29 Jan 2018 16:20:02 +0000</updated>
                            <resolved>Mon, 8 Jan 2018 03:57:24 +0000</resolved>
                                                                    <component>WiredTiger</component>
                                        <votes>2</votes>
                                    <watches>12</watches>
                                                                                                                <comments>
                            <comment id="1764082" author="bpot" created="Wed, 3 Jan 2018 17:39:57 +0000"  >&lt;p&gt;Great! We are already on 3.6.1 and that issue has gone away.&lt;/p&gt;

&lt;p&gt;We are now seeing a different performance issue trigger when cache isn&apos;t maxed out. I&apos;ll open a new ticket for that though once I can repro again and collect diagnostic information.&lt;/p&gt;</comment>
                            <comment id="1763614" author="alexander.gorrod" created="Wed, 3 Jan 2018 04:18:04 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bpot&quot; class=&quot;user-hover&quot; rel=&quot;bpot&quot;&gt;bpot&lt;/a&gt; Sorry for the long delay in getting back to you and thanks for uploading the data. I&apos;ve taken a look at the data, and the symptoms you see correspond to some that were fixed in &lt;a href=&quot;https://jira.mongodb.org/browse/WT-3079&quot; title=&quot;Make sure eviction visits all trees&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-3079&quot;&gt;&lt;del&gt;WT-3079&lt;/del&gt;&lt;/a&gt;, that fix is in the latest version of MongoDB 3.6, but hasn&apos;t yet landed in a version of MongoDB 3.4. I&apos;d recommend you upgrading to the next version of MongoDB 3.4 when it becomes available, as I expect it to improve the behavior for you.&lt;/p&gt;

&lt;p&gt;In particular your workload has a collection that uses the majority of the cache, and the fix in &lt;a href=&quot;https://jira.mongodb.org/browse/WT-3079&quot; title=&quot;Make sure eviction visits all trees&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-3079&quot;&gt;&lt;del&gt;WT-3079&lt;/del&gt;&lt;/a&gt; meant we more fairly added pages to the LRU queue in cases like that.&lt;/p&gt;</comment>
                            <comment id="1682407" author="bpot" created="Tue, 26 Sep 2017 19:13:47 +0000"  >&lt;p&gt;I&apos;ve uploaded logs for about a 10 minute period where we were experiencing the issue. Let me know if there is any additional information which would be helpful.&lt;/p&gt;

&lt;p&gt;Thanks for looking into this.&lt;/p&gt;</comment>
                            <comment id="1681585" author="alexander.gorrod" created="Tue, 26 Sep 2017 04:18:00 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bpot&quot; class=&quot;user-hover&quot; rel=&quot;bpot&quot;&gt;bpot&lt;/a&gt; Thanks for uploading the logs - we have investigated the data. We haven&apos;t seen similar behavior in other workloads - it would help us understand the behavior on your system if you can share additional diagnostic information while the system is in the described state. It&apos;s possible that the additional information gathering will have some affect on throughput - so please monitor carefully when enabling.&lt;/p&gt;

&lt;p&gt;How you enable the logging depends on how you administer your database, the instructions I give below work through the mongodb shell.&lt;/p&gt;

&lt;p&gt;Enable the additional logging using:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;db.getSiblingDB(&#8216;admin&#8217;).runCommand( { setParameter: 1, wiredTigerEngineRuntimeConfig : &#8220;statistics=[fast,cache_walk],statistics_log=(wait=10,json=false,sources=[file:])&#8221; } )&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Allow the machine to run for approximately 10 minutes with this setting enabled. Disable the setting by running:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;db.getSiblingDB(&#8216;admin&#8217;).runCommand( { setParameter: 1, wiredTigerEngineRuntimeConfig : &#8220;statistics=[fast,cache_walk],statistics_log=(wait=0,json=false,sources=[file:])&#8221; } )&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Then please upload the following for the time period corresponding to when the additional logging is enabled:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;The &lt;tt&gt;mongod log&lt;/tt&gt;&lt;/li&gt;
	&lt;li&gt;The &lt;tt&gt;diagnostic.data&lt;/tt&gt; subdirectory of the database directory&lt;/li&gt;
	&lt;li&gt;Any files that match the pattern &lt;tt&gt;WiredTigerStat.XX.XX&lt;/tt&gt; that exist in the database directory. Where &lt;tt&gt;X&lt;/tt&gt; corresponds to a numerical digit.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;The additional information enabled here doesn&apos;t track application specific content - it purely tracks abstract database operations, so there shouldn&apos;t be any privacy issues.&lt;/p&gt;</comment>
                            <comment id="1676726" author="bpot" created="Tue, 19 Sep 2017 17:10:09 +0000"  >&lt;p&gt;Hi,&lt;/p&gt;

&lt;p&gt;I don&apos;t have the logs from the original time period but I&apos;ve uploaded redacted logs and diagnostic data for a more recent occurrence.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Bob&lt;/p&gt;</comment>
                            <comment id="1675983" author="thomas.schubert" created="Mon, 18 Sep 2017 21:43:23 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bpot&quot; class=&quot;user-hover&quot; rel=&quot;bpot&quot;&gt;bpot&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Thanks for providing the diagnostic.data, we&apos;re still examining it to better understand what is going on, but an initial review suggests that it would be helpful if you could provide the complete log files covering the same time period so we can correlate these events. If there is sensitive material in the logs, would you please redact it using the same procedure as in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-22642&quot; title=&quot;WiredTiger engine resync stalls with a lot of tables/indexes&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-22642&quot;&gt;&lt;del&gt;SERVER-22642&lt;/del&gt;&lt;/a&gt;?&lt;/p&gt;

&lt;p&gt;Kind regards,&lt;br/&gt;
Kelsey&lt;/p&gt;</comment>
                            <comment id="1675929" author="bpot" created="Mon, 18 Sep 2017 20:59:14 +0000"  >&lt;p&gt;Thanks. I&apos;ve uploaded the data.&lt;/p&gt;</comment>
                            <comment id="1675890" author="thomas.schubert" created="Mon, 18 Sep 2017 20:36:23 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bpot&quot; class=&quot;user-hover&quot; rel=&quot;bpot&quot;&gt;bpot&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;I&apos;ve created a secure &lt;a href=&quot;https://10gen-httpsupload.s3.amazonaws.com/upload_forms/eed29198-f991-446f-b71b-cb4c7f6c93d1.html&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;upload portal&lt;/a&gt; for you to use to provide an archive of the &lt;tt&gt;diagnostic.data&lt;/tt&gt; directory.&lt;/p&gt;

&lt;p&gt;Thank you,&lt;br/&gt;
Kelsey&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="165802" name="Screen Shot 2017-09-18 at 5.36.27 PM.png" size="475826" author="kelsey.schubert@mongodb.com" created="Mon, 18 Sep 2017 21:39:28 +0000"/>
                            <attachment id="165919" name="Screen Shot 2017-09-19 at 4.57.59 PM.png" size="279905" author="kelsey.schubert@mongodb.com" created="Tue, 19 Sep 2017 21:35:29 +0000"/>
                            <attachment id="165858" name="eviction.png" size="497538" author="bruce.lucas@mongodb.com" created="Tue, 19 Sep 2017 14:06:48 +0000"/>
                            <attachment id="165791" name="mongostat.txt" size="3759" author="bpot" created="Mon, 18 Sep 2017 20:08:57 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>8.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 18 Sep 2017 20:36:23 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        6 years, 6 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>backlog-server-pm</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            6 years, 6 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>alexander.gorrod@mongodb.com</customfieldvalue>
            <customfieldvalue>bpot</customfieldvalue>
            <customfieldvalue>kelsey.schubert@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htf2nj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|ht6wmn:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[kelsey.schubert@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hteoqv:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>