<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:17:00 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-48395] Extended stalls during heavy insert workload</title>
                <link>https://jira.mongodb.org/browse/SERVER-48395</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;While working with the repro for &lt;a href=&quot;https://jira.mongodb.org/browse/WT-6175&quot; title=&quot;tcmalloc fragmentation is worse in 4.4 with durable history&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-6175&quot;&gt;&lt;del&gt;WT-6175&lt;/del&gt;&lt;/a&gt; I noticed that there were extended stalls during the insert phase.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/262366/262366_stalls.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;The stalls seem to end with the start of the next checkpoint&lt;/li&gt;
	&lt;li&gt;With checkpoints disabled the stalls lasted as long as 10 minutes&lt;/li&gt;
	&lt;li&gt;During the stalls the log reports operations that took the entire duration of the stall to complete&lt;/li&gt;
	&lt;li&gt;They appear to have something to do with page splits.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;FTDC, logs, and repro script attached. The repro creates two collections of 5 GB each with a 5 GB cache, using 50 client threads on a machine with 24 cpus.&lt;/p&gt;</description>
                <environment></environment>
        <key id="1361790">SERVER-48395</key>
            <summary>Extended stalls during heavy insert workload</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="sulabh.mahajan@mongodb.com">Sulabh Mahajan</assignee>
                                    <reporter username="bruce.lucas@mongodb.com">Bruce Lucas</reporter>
                        <labels>
                            <label>KP44</label>
                    </labels>
                <created>Mon, 25 May 2020 14:48:19 +0000</created>
                <updated>Sun, 29 Oct 2023 22:07:49 +0000</updated>
                            <resolved>Wed, 15 Jul 2020 19:32:08 +0000</resolved>
                                    <version>4.4.0-rc6</version>
                    <version>4.4.0-rc7</version>
                                    <fixVersion>4.4.0-rc13</fixVersion>
                                    <component>Storage</component>
                                        <votes>0</votes>
                                    <watches>18</watches>
                                                                                                                <comments>
                            <comment id="3286846" author="bruce.lucas@10gen.com" created="Wed, 15 Jul 2020 19:31:31 +0000"  >&lt;p&gt;Yes, I can confirm that the stalls are gone. Thanks!&lt;/p&gt;</comment>
                            <comment id="3281566" author="sulabh.mahajan" created="Mon, 13 Jul 2020 03:20:45 +0000"  >&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/269588/269588_server-resolved.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;I re-ran the test today on RC13, and see no stalls. If Bruce confirms that this issue is fixed, we will close this ticket.&lt;/p&gt;</comment>
                            <comment id="3281440" author="sulabh.mahajan" created="Sun, 12 Jul 2020 14:11:51 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt;, the issues that we identified with this ticket and with &lt;a href=&quot;https://jira.mongodb.org/browse/WT-6444&quot; title=&quot;Abort a transaction if it is force evicting and oldest&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-6444&quot;&gt;&lt;del&gt;WT-6444&lt;/del&gt;&lt;/a&gt; have all been merged into v4.4. Could you please verify that the workload attached to this ticket is not stalling anymore (than 4.2).&lt;/p&gt;</comment>
                            <comment id="3225338" author="alex.cameron" created="Fri, 26 Jun 2020 10:49:17 +0000"  >&lt;p&gt;I wasn&apos;t able to find anything from our usage of the &lt;tt&gt;last_eviction_timestamp&lt;/tt&gt;. I had a quick look at what we&apos;re doing with &lt;tt&gt;last_eviction_timestamp&lt;/tt&gt; since that&apos;s the check that is causing is trouble. We&apos;re using it the same was as in &lt;tt&gt;v4.2&lt;/tt&gt; so the issue isn&apos;t obvious.&lt;/p&gt;

&lt;p&gt;I had a bit of a suspicion that it was somehow related to the problem of not pinning ids for history store cursors (even though this is saving the pinned timestamp). I tried running our repro script even with my patch on top of &lt;a href=&quot;https://jira.mongodb.org/browse/WT-6462&quot; title=&quot;Use read uncommitted isolation level for history store operations&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-6462&quot;&gt;&lt;del&gt;WT-6462&lt;/del&gt;&lt;/a&gt; in &lt;a href=&quot;https://github.com/wiredtiger/wiredtiger/pull/5858&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/wiredtiger/wiredtiger/pull/5858&lt;/a&gt; but still saw the erratic performance.&lt;/p&gt;</comment>
                            <comment id="3223573" author="sulabh.mahajan" created="Thu, 25 Jun 2020 12:29:11 +0000"  >&lt;p&gt;Copying the comment from &lt;a href=&quot;https://jira.mongodb.org/browse/WT-6444&quot; title=&quot;Abort a transaction if it is force evicting and oldest&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-6444&quot;&gt;&lt;del&gt;WT-6444&lt;/del&gt;&lt;/a&gt;:&lt;/p&gt;

&lt;p&gt;Update:&lt;/p&gt;

&lt;p&gt;The issue here seems to be a manifestation of the same cause as &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-48395&quot; title=&quot;Extended stalls during heavy insert workload&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-48395&quot;&gt;&lt;del&gt;SERVER-48395&lt;/del&gt;&lt;/a&gt;. I have not root caused it completely yet, but there are some observations so far:&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;The insert workload populates pages in such a way that a small number of pages get all the appends and quickly grow in size&lt;/li&gt;
	&lt;li&gt;Once the page reaches the size that pages are generally split at (80% of the max page, which is 10 MB for collections and 5 MB for indexes),  the pages are forcibly evicted by the application threads before they can do more inserts.&lt;/li&gt;
	&lt;li&gt;This is expected and we detect an append workload and try to split such that the last inserts of the page split and go to a new page, where the threads can continue inserting into.&lt;/li&gt;
	&lt;li&gt;I noticed that a lot of the above result in failures because split needs access to the parent page and we fail to get that access for some reason.&lt;/li&gt;
	&lt;li&gt;While we fail to split we still continue putting more inserts and let the pages grow, but they are only allowed to grow to a certain extent. Eventually, we start to see very slow eviction of these new pages, which I believe is the page repeatedly being read and evicted, resulting in a severe slowdown of the workload. While this happens as part of the insert transaction, the transaction gets pinned as well.&lt;/li&gt;
	&lt;li&gt;While we are in this degraded state we are unable to evict, and we repeatedly do 1 - 1 replacement of the pages in memory. I think here we are spending a lot of time trying to evict, but not making any real progress.&lt;/li&gt;
	&lt;li&gt;We have a portion of code that could detect that eviction is not making progress and likely holding the transactions pinned, so we fail and rollback the transaction. The rollback then results in resolving the situation temporarily, till we get into the same degraded state again.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;I am still working on figuring out a few things here:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;Why are we unable to get lock to the parent pages to do more effective forced in-memory splits&lt;/li&gt;
	&lt;li&gt;What leads to that degraded state where eviction ends up replacing page by page without making any real progress.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;There are a lot of moving parts here because the stall generally happens around 4% dirty, which is less than when eviction starts (5%). As we stall and fail to evict (pinned transactions) the dirty content rises and crosses 5% which further triggers a different behaviour of eviction. Trying to catch the stall before the eviction target of 5% is reached has been challenging. We are also working on a standalone wiredtiger workload to stimulate the issue, one which has less moving parts and can be controlled easily to debug the problem.&lt;/p&gt;</comment>
                            <comment id="3213486" author="brian.lane" created="Thu, 18 Jun 2020 02:15:01 +0000"  >&lt;p&gt;Thanks Sulabh for investigating.&lt;/p&gt;

&lt;p&gt;I agree we should fix this, but not block 4.4 GA on it.  I am ok to see this one moved to 4.5 required.&lt;/p&gt;</comment>
                            <comment id="3211964" author="alexander.gorrod" created="Wed, 17 Jun 2020 11:54:37 +0000"  >&lt;p&gt;To characterize this issue somewhat. The test program has 50 threads inserting batches of 10k empty documents in parallel. That is a worst case scenario for testing how well WiredTiger can get exclusive access to hot pages in append workloads. We have code to efficiently split the front of a busy page and allow new updates to come into that page while we clean up the old page. If there is enough activity remaining on the old page (which can happen when there are lots of threads contending to append), we can end up in a situation where those threads that are still updating the previous page wait for it to be force evicted, and then wait for their part of it to be read back in before being able to proceed.&lt;/p&gt;

&lt;p&gt;It&apos;s something we need to understand and address, but it&apos;s not something MongoDB users rely on at the scale used by this test as is indicated by the problem being very significantly mitigated when the oplog is enabled.&lt;/p&gt;

&lt;p&gt;I think we need to fix this, but after talking it through with &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sulabh.mahajan&quot; class=&quot;user-hover&quot; rel=&quot;sulabh.mahajan&quot;&gt;sulabh.mahajan&lt;/a&gt; today, I&apos;m not sure it should be a 4.4.0 release blocker.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt; or &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=kelsey.schubert&quot; class=&quot;user-hover&quot; rel=&quot;kelsey.schubert&quot;&gt;kelsey.schubert&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=pasette&quot; class=&quot;user-hover&quot; rel=&quot;pasette&quot;&gt;pasette&lt;/a&gt; and &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=brian.lane&quot; class=&quot;user-hover&quot; rel=&quot;brian.lane&quot;&gt;brian.lane&lt;/a&gt;. Could you please let me know if you agree?&lt;/p&gt;</comment>
                            <comment id="3211776" author="sulabh.mahajan" created="Wed, 17 Jun 2020 07:28:27 +0000"  >&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/266160/266160_with-oplog.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/266161/266161_with-oplog-closeup.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;I ran the same test with oplog enabled. I ran 4.2 to compare as well. The problem goes away when the node is a part of replica set. We still have some dip in insert performance, but that happens during a checkpoint, not before it. The same dip is also observed in 4.2, so it is not a new problem in 4.4. The magnitude of dip might be larger than 4.2, but the test is not designed to be conclusive in that observation.&lt;/p&gt;

&lt;p&gt;Why we have stalls without oplog:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;From statistics the best guess is that we are force-evicting pages more than before. There are new conditions in 4.4 around when a page is considered evictable (some of the content has been moved to history store?) which were not the same as in 4.2&lt;/li&gt;
	&lt;li&gt;Since we evict the pages that the other threads are still populating, the other threads get stalled waiting for the page to be reconciled, evicted and then read back again, bringing the performance down.&lt;/li&gt;
	&lt;li&gt;Since we do not force evict when a checkpoint is going on,  the insert rate goes up when a checkpoint starts.&lt;/li&gt;
&lt;/ul&gt;
</comment>
                            <comment id="3209856" author="sulabh.mahajan" created="Tue, 16 Jun 2020 14:12:47 +0000"  >&lt;blockquote&gt;
&lt;p&gt;That&apos;s interesting - can you tell whether the maximum page size at eviction was growing as large with 4.2?&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt; &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/266067/266067_4.2-pic.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;In 4.2, we do get to a similarly huge page by the time we evict. We do not see stalls or a correlation to the forced eviction / in-memory page splits.&lt;/p&gt;

&lt;p&gt;I have tried disabling force evicts in 4.4, but I think I have not disabled all the paths to force evict and I still can&apos;t get away with stalls.&lt;/p&gt;

&lt;blockquote&gt;
&lt;p&gt;We should figure out if something has changed here in terms of effectively getting exclusive access to the page when we start wanting it.&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;Will check that tomorrow.&lt;/p&gt;</comment>
                            <comment id="3209364" author="alexander.gorrod" created="Tue, 16 Jun 2020 06:06:01 +0000"  >&lt;p&gt;I suspect what is happening here is that during checkpoints, we know we can&apos;t force evict the page so we don&apos;t bother trying (that&apos;s the call to &lt;tt&gt;_&lt;em&gt;wt_btree_can_evict_dirty&lt;/tt&gt; in &lt;tt&gt;&lt;/em&gt;_wt_page_can_evict&lt;/tt&gt;). So during checkpoint we allow a lot of updates to come into the page, and the page grows to be large.&lt;/p&gt;

&lt;p&gt;Once checkpoint finishes, we keep trying to force evict the page, but aren&apos;t being successful. The attempts to force evict the page are causing throughput to slow down significantly. We should figure out if something has changed here in terms of effectively getting exclusive access to the page when we start wanting it.&lt;/p&gt;</comment>
                            <comment id="3209358" author="alexander.gorrod" created="Tue, 16 Jun 2020 05:51:27 +0000"  >&lt;p&gt;That&apos;s interesting - can you tell whether the maximum page size at eviction was growing as large with 4.2?&lt;/p&gt;</comment>
                            <comment id="3209339" author="sulabh.mahajan" created="Tue, 16 Jun 2020 04:26:06 +0000"  >&lt;p&gt; &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/266033/266033_stall-in-mem-split.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;Update:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;I am able to reproduce with the  attached script&lt;/li&gt;
	&lt;li&gt;I have run multiple times with per file FTDC collection as well to analyse data. Sometimes there are stalls not correlating to the checkpoint. Start of the checkpoint &quot;helps&quot; release the stall&lt;/li&gt;
	&lt;li&gt;After spending a lot of time looking at FTDC stats, I think it is because of the pages that are getting the inserts growing large and being forcibly evicted. While being force evicted by the application threads, then the threads are throttled to do more work.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;The above theory is not conclusive yet, I am still working on the ticket and will post more updates&lt;/p&gt;</comment>
                            <comment id="3142341" author="bruce.lucas@10gen.com" created="Fri, 29 May 2020 11:36:24 +0000"  >&lt;p&gt;Thanks for looking at this.&lt;/p&gt;
&lt;blockquote&gt;
&lt;p&gt;I think it&apos;s probably the other way around - checkpoints are waiting for a reconciliation to finish before they can get started.&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;Not so sure about that - the checkpoints are happening regularly with exactly the expected 60 seconds between the end of one and the start of the next. Also without the checkpoints the stalls last much longer. Naively, it&apos;s as if the start of the checkpoint is somehow interrupting the the slow reconciliation process  that you describe.&lt;/p&gt;

&lt;p&gt;I seem to recall seeing this behavior before, where a stall ended with an on-schedule checkpoint, e.g. &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-16938&quot; title=&quot;60-second stall between checkpoints under WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-16938&quot;&gt;&lt;del&gt;SERVER-16938&lt;/del&gt;&lt;/a&gt;, also associated with highly concurrent workload.&lt;/p&gt;</comment>
                            <comment id="3142189" author="alexander.gorrod" created="Fri, 29 May 2020 08:18:17 +0000"  >&lt;blockquote&gt;&lt;p&gt;The stalls seem to end with the start of the next checkpoint&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;I think it&apos;s probably the other way around - checkpoints are waiting for a reconciliation to finish before they can get started.&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;With checkpoints disabled the stalls lasted as long as 10 minutes&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;It&apos;s just a theory, but I &lt;b&gt;think&lt;/b&gt; what is going on here, is that the workload has 50 threads all contending to insert things at the head of a collection. We have historically had some troubles maintaining throughput in this workload, and have a set of heuristics related to getting exclusive access to a hot page, splitting content off it, and allowing the appends to continue.&lt;/p&gt;

&lt;p&gt;We start trying to get exclusive access to do what we term an in-memory split when a page gets to 8MB (80% of memory_page_max) in cache. If we haven&apos;t succeeded in getting exclusive access when the page reaches 10MB we get more and more assertive (disrupting application threads usual control flow) to try to get exclusive access. If it&apos;s reached that threshold we kind of give up on in-memory splits and just reconcile the page. In this workload, by the time we get exclusive access to the page it&apos;s grown to 70MB. It takes a long time for the thread reconciling that page to process all the content - even more because we&apos;ve got some inefficiencies at the moment about how reconciliation checks for history that needs to be cleaned up (e.g: &lt;a href=&quot;https://jira.mongodb.org/browse/WT-5769&quot; title=&quot;Search history store can potentially walk the whole history store tree&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-5769&quot;&gt;&lt;del&gt;WT-5769&lt;/del&gt;&lt;/a&gt;).&lt;/p&gt;

&lt;p&gt;We&apos;ve also changed some of the restrictions about when a page can be evicted and what happens to it. It used to be that we didn&apos;t allow eviction of a page with updates in the range of active transaction IDs unless the cache overflow mechanism was in use. With durable history we&apos;ve relaxed some of that constraint, and it&apos;s probably hurting here.&lt;/p&gt;

&lt;p&gt;&lt;b&gt;tl;dr:&lt;/b&gt; The stalls here have hallmarks of multi-threaded append contention, the symptom is exacerbated by some inefficiencies in reconciliation and eviction scheduling changes that are associated with durable history. We are working on the reconciliation inefficiencies in &lt;a href=&quot;https://jira.mongodb.org/browse/WT-5769&quot; title=&quot;Search history store can potentially walk the whole history store tree&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-5769&quot;&gt;&lt;del&gt;WT-5769&lt;/del&gt;&lt;/a&gt;, and can schedule work to look at the scheduling problem if an issue remains.&lt;/p&gt;</comment>
                            <comment id="3106556" author="bruce.lucas@10gen.com" created="Tue, 26 May 2020 11:38:17 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=michael.cahill&quot; class=&quot;user-hover&quot; rel=&quot;michael.cahill&quot;&gt;michael.cahill&lt;/a&gt;, no, I reran this (the insert portion of that test only) against rc6. Looks like I forgot to attach ftdc and logs; attached now.&lt;/p&gt;</comment>
                            <comment id="3106199" author="michael.cahill" created="Mon, 25 May 2020 22:16:21 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt;, was this against 4.4.0-rc3, as per your most recent comment in &lt;a href=&quot;https://jira.mongodb.org/browse/WT-6175&quot; title=&quot;tcmalloc fragmentation is worse in 4.4 with durable history&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-6175&quot;&gt;&lt;del&gt;WT-6175&lt;/del&gt;&lt;/a&gt;?&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                            <outwardlinks description="depends on">
                                        <issuelink>
            <issuekey id="1384030">WT-6444</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1395297">WT-6484</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1397548">WT-6488</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="1347520">WT-6175</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1384030">WT-6444</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="1361806">SERVER-48396</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="266067" name="4.2-pic.png" size="146011" author="sulabh.mahajan@mongodb.com" created="Tue, 16 Jun 2020 14:10:10 +0000"/>
                            <attachment id="262424" name="insert.tgz" size="1408347" author="bruce.lucas@mongodb.com" created="Tue, 26 May 2020 11:37:25 +0000"/>
                            <attachment id="262365" name="repro.sh" size="883" author="bruce.lucas@mongodb.com" created="Mon, 25 May 2020 14:47:57 +0000"/>
                            <attachment id="269588" name="server-resolved.png" size="130300" author="sulabh.mahajan@mongodb.com" created="Mon, 13 Jul 2020 03:19:48 +0000"/>
                            <attachment id="266033" name="stall-in-mem-split.png" size="107651" author="sulabh.mahajan@mongodb.com" created="Tue, 16 Jun 2020 04:21:25 +0000"/>
                            <attachment id="262366" name="stalls.png" size="275974" author="bruce.lucas@mongodb.com" created="Mon, 25 May 2020 14:47:45 +0000"/>
                            <attachment id="266161" name="with-oplog-closeup.png" size="95575" author="sulabh.mahajan@mongodb.com" created="Wed, 17 Jun 2020 07:16:28 +0000"/>
                            <attachment id="266160" name="with-oplog.png" size="112197" author="sulabh.mahajan@mongodb.com" created="Wed, 17 Jun 2020 07:16:27 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>16.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>4.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 25 May 2020 22:16:21 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        3 years, 30 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[<s><a href='https://jira.mongodb.org/browse/WT-6484'>WT-6484</a></s>, <s><a href='https://jira.mongodb.org/browse/WT-6444'>WT-6444</a></s>, <s><a href='https://jira.mongodb.org/browse/WT-6488'>WT-6488</a></s>]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            3 years, 30 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_16465" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Linked BF Score</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>alexander.gorrod@mongodb.com</customfieldvalue>
            <customfieldvalue>alex.cameron@mongodb.com</customfieldvalue>
            <customfieldvalue>brian.lane@mongodb.com</customfieldvalue>
            <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>michael.cahill@mongodb.com</customfieldvalue>
            <customfieldvalue>sulabh.mahajan@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hxmwlj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr54xr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="4001">Storage - Ra 2020-06-15</customfieldvalue>
    <customfieldvalue id="4038">Storage - Ra 2020-06-29</customfieldvalue>
    <customfieldvalue id="4099">Storage - Ra 2020-07-13</customfieldvalue>
    <customfieldvalue id="4100">Storage - Ra 2020-07-27</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10555" key="com.atlassian.jira.plugin.system.customfieldtypes:float">
                        <customfieldname>Story Points</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>5.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hxmiuv:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>