<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:38:50 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-35103] Checkpoint creates unevictable clean content</title>
                <link>https://jira.mongodb.org/browse/SERVER-35103</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Attached repro script does the following&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;Inserts 1M documents with 10 indexes over fields that are random strings&lt;/li&gt;
	&lt;li&gt;Stops one secondary then removes all documents via one of the indexes, therefore in random order.&lt;/li&gt;
	&lt;li&gt;Restarts that secondary. While it is catching up (after the script completes), due to &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-34938&quot; title=&quot;Secondary slowdown or hang due to content pinned in cache by single oplog batch&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-34938&quot;&gt;&lt;del&gt;SERVER-34938&lt;/del&gt;&lt;/a&gt;, batches will be large and take several minutes to complete, so multiple checkpoints will run while timestamps are pinning content in cache. It appears that additional unevictable clean content is generated at each checkpoint, exacerbating the effects of &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-34938&quot; title=&quot;Secondary slowdown or hang due to content pinned in cache by single oplog batch&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-34938&quot;&gt;&lt;del&gt;SERVER-34938&lt;/del&gt;&lt;/a&gt;.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/187202/187202_repro.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;batch boundaries are at A, E, F&lt;/li&gt;
	&lt;li&gt;at each checkpoint (e.g B, C, D) the amount of clean content is increased, and this is presumably unevictable because the oldest timestamp is only updated between batches&lt;/li&gt;
	&lt;li&gt;at batch boundaries when oldest timestamp is advanced the cache content is reduced, presumably because it becomes evictable&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Above was run on a machine with 24 cpus and a lot of memory, but the script uses numactl to limit each instance to two cpus and cache to 4 GB (during setup) and 1 GB (during recovery), so this should be runnable on a more modest machine.&lt;/p&gt;</description>
                <environment></environment>
        <key id="546808">SERVER-35103</key>
            <summary>Checkpoint creates unevictable clean content</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="bruce.lucas@mongodb.com">Bruce Lucas</assignee>
                                    <reporter username="bruce.lucas@mongodb.com">Bruce Lucas</reporter>
                        <labels>
                            <label>nonnyc</label>
                            <label>storage-engines</label>
                    </labels>
                <created>Sat, 19 May 2018 14:30:21 +0000</created>
                <updated>Thu, 23 Aug 2018 14:02:45 +0000</updated>
                            <resolved>Thu, 23 Aug 2018 14:02:45 +0000</resolved>
                                    <version>3.6.3</version>
                    <version>3.6.4</version>
                                                    <component>Storage</component>
                                        <votes>0</votes>
                                    <watches>9</watches>
                                                                                                                <comments>
                            <comment id="1983756" author="greg.mckeon" created="Thu, 23 Aug 2018 14:02:30 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt; was there a commit on this ticket? If so, can you link it? I don&apos;t see one in the codebase, so I&apos;m setting the resolution as &quot;Done&quot;.&lt;/p&gt;</comment>
                            <comment id="1902968" author="alexander.gorrod" created="Tue, 29 May 2018 04:07:47 +0000"  >&lt;p&gt;I ran the workload attached to this ticket against the 3.6.3 and 3.6.5 versions of MongoDB, and I can reproduce the behavior you describe with the earlier version but not the latter. The following picture demonstrates the difference:&lt;/p&gt;

&lt;p&gt; &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/187761/187761_s35103.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;You can see that the newer version shows the same saw pattern of cache usage related to checkpoints, but the total cache usage never goes above 82% - which is the threshold for the eviction server starting to work on making more space available. The dirty fill ratio is steady at 20% (max) for almost the entire run.&lt;/p&gt;

&lt;p&gt;I tried an experiment with the latest version of the 3.6 code, where I allowed a higher proportion of the cache to be dirty via &lt;tt&gt;--wiredTigerEngineConfigString=eviction_dirty_trigger=75,eviction_dirty_target=50&lt;/tt&gt;, with the theory that if clean content were actually pinned we would see the cache get stuck. I didn&apos;t observe that - the workload effectively continued even when there was less space for clean content in the cache:&lt;/p&gt;

&lt;p&gt; &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/187762/187762_s35103_high_dirty.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;I also added a new statistic during this investigation which tracks the case &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=michael.cahill&quot; class=&quot;user-hover&quot; rel=&quot;michael.cahill&quot;&gt;michael.cahill&lt;/a&gt; was concerned we had wrong - which was that WiredTiger might not consider clean pages with history for eviction (even though it can use the lookaside mechanism to reclaim the space). That statistic is growing in all of these workloads - which indicates that those pages would be chosen for eviction if there was enough cache pressure.&lt;/p&gt;

&lt;p&gt;Given that the behavior is improved in more recent versions, and the particular code that was of concern is working properly I don&apos;t think there is anything more to do here. I&apos;m going to assign this back to you &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt; rather than closing outright, in case you&apos;d like more information or a different follow up.&lt;/p&gt;</comment>
                            <comment id="1901591" author="alexander.gorrod" created="Fri, 25 May 2018 06:18:46 +0000"  >&lt;p&gt;I have done some investigation into the behavior described and can&apos;t see any changes in WiredTiger that have been done since 3.6.4 that are likely to fix this issue. I&#160;am running the reproducer and will do more analysis into how we can mitigate the behavior.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="494491">SERVER-33191</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="542268">SERVER-34938</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="187201" name="metrics.2018-05-19T13-40-35Z-00000" size="808054" author="bruce.lucas@mongodb.com" created="Sat, 19 May 2018 14:30:17 +0000"/>
                            <attachment id="187202" name="repro.png" size="136520" author="bruce.lucas@mongodb.com" created="Sat, 19 May 2018 14:30:09 +0000"/>
                            <attachment id="187203" name="repro.sh" size="3063" author="bruce.lucas@mongodb.com" created="Sat, 19 May 2018 14:29:58 +0000"/>
                            <attachment id="187761" name="s35103.png" size="195409" author="alexander.gorrod@mongodb.com" created="Tue, 29 May 2018 03:56:07 +0000"/>
                            <attachment id="187762" name="s35103_high_dirty.png" size="103369" author="alexander.gorrod@mongodb.com" created="Tue, 29 May 2018 04:03:19 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>3.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 25 May 2018 06:18:46 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        5 years, 24 weeks, 6 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>greg.mckeon@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            5 years, 24 weeks, 6 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>alexander.gorrod@mongodb.com</customfieldvalue>
            <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>greg.mckeon@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htyjvr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|htnshr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="2349">Storage Non-NYC 2018-06-04</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[bruce.lucas@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hty653:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>