<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:32:37 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-33191] Cache-full hangs on 3.6</title>
                <link>https://jira.mongodb.org/browse/SERVER-33191</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;We have seen a couple of cases in the field where under heavy cache pressure a node can get stuck with the WT cache full. In the cases seen so far the cache pressure has been due to replica set lag, which causes the majority commit point to lag, creating cache pressure.&lt;/p&gt;

&lt;p&gt;So far we don&apos;t have a reproducer and don&apos;t have FTDC data covering the inception of the issue (&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-32876&quot; title=&quot;Don&amp;#39;t stall ftdc due to WT cache full&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-32876&quot;&gt;&lt;del&gt;SERVER-32876&lt;/del&gt;&lt;/a&gt;) so this ticket for now is a placeholder to gather information.&lt;/p&gt;</description>
                <environment></environment>
        <key id="494491">SERVER-33191</key>
            <summary>Cache-full hangs on 3.6</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="benety.goh@mongodb.com">Benety Goh</assignee>
                                    <reporter username="bruce.lucas@mongodb.com">Bruce Lucas</reporter>
                        <labels>
                    </labels>
                <created>Thu, 8 Feb 2018 15:54:49 +0000</created>
                <updated>Sun, 29 Oct 2023 22:34:59 +0000</updated>
                            <resolved>Fri, 27 Jul 2018 14:49:49 +0000</resolved>
                                    <version>3.6.2</version>
                    <version>3.6.5</version>
                                    <fixVersion>3.6.7</fixVersion>
                    <fixVersion>4.0.1</fixVersion>
                    <fixVersion>4.1.2</fixVersion>
                                    <component>Storage</component>
                                        <votes>1</votes>
                                    <watches>22</watches>
                                                                                                                <comments>
                            <comment id="1958508" author="benety.goh" created="Fri, 27 Jul 2018 14:49:49 +0000"  >&lt;p&gt;Reproduction script shows that the stuck cache issue was resolved between 3.6.5 and 3.6.6. There is a reported performance regression reported in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-36221&quot; title=&quot;[3.6] Performance regression on small updates to large documents&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-36221&quot;&gt;&lt;del&gt;SERVER-36221&lt;/del&gt;&lt;/a&gt; that we will continue to investigate.&lt;/p&gt;</comment>
                            <comment id="1952730" author="xgen-internal-githook" created="Fri, 20 Jul 2018 20:27:39 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Benety Goh&apos;, &apos;email&apos;: &apos;benety@mongodb.com&apos;, &apos;username&apos;: &apos;benety&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-33191&quot; title=&quot;Cache-full hangs on 3.6&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-33191&quot;&gt;&lt;del&gt;SERVER-33191&lt;/del&gt;&lt;/a&gt; move wt_cache_full.js from jstests/replsets/ to jstests/noPassthrough/&lt;/p&gt;

&lt;p&gt;(cherry picked from commit f5035fab356299fe6cffc783dbe9ccfab99fe069)&lt;br/&gt;
Branch: v3.6&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/13f9d42c1be08299c191eac13beb70b570843a48&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/13f9d42c1be08299c191eac13beb70b570843a48&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1952728" author="xgen-internal-githook" created="Fri, 20 Jul 2018 20:27:37 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Benety Goh&apos;, &apos;email&apos;: &apos;benety@mongodb.com&apos;, &apos;username&apos;: &apos;benety&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-33191&quot; title=&quot;Cache-full hangs on 3.6&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-33191&quot;&gt;&lt;del&gt;SERVER-33191&lt;/del&gt;&lt;/a&gt; add test to fill wiredtiger cache during steady state oplog replication&lt;/p&gt;

&lt;p&gt;(cherry picked from commit 88cea24944cf277759b74b18878a0b4c0d89686e)&lt;br/&gt;
Branch: v3.6&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/258db0da725579e23ac41a9d6c3e0ac82a7d688f&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/258db0da725579e23ac41a9d6c3e0ac82a7d688f&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1952727" author="xgen-internal-githook" created="Fri, 20 Jul 2018 20:27:35 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Benety Goh&apos;, &apos;email&apos;: &apos;benety@mongodb.com&apos;, &apos;username&apos;: &apos;benety&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-33191&quot; title=&quot;Cache-full hangs on 3.6&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-33191&quot;&gt;&lt;del&gt;SERVER-33191&lt;/del&gt;&lt;/a&gt; guard mongo test program startup options against duplicate wiredtiger flags&lt;/p&gt;

&lt;p&gt;(cherry picked from commit eaf551c3aa01eb80b48bc7601aa288eed464532c)&lt;br/&gt;
Branch: v3.6&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/125de5067cf8042247f5bb14f7db2f36d36aab14&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/125de5067cf8042247f5bb14f7db2f36d36aab14&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1952651" author="xgen-internal-githook" created="Fri, 20 Jul 2018 19:35:34 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Benety Goh&apos;, &apos;email&apos;: &apos;benety@mongodb.com&apos;, &apos;username&apos;: &apos;benety&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-33191&quot; title=&quot;Cache-full hangs on 3.6&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-33191&quot;&gt;&lt;del&gt;SERVER-33191&lt;/del&gt;&lt;/a&gt; move wt_cache_full.js from jstests/replsets/ to jstests/noPassthrough/&lt;/p&gt;

&lt;p&gt;(cherry picked from commit f5035fab356299fe6cffc783dbe9ccfab99fe069)&lt;br/&gt;
Branch: v4.0&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/f6a183706a304e0260678e1e754347fbc0f15793&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/f6a183706a304e0260678e1e754347fbc0f15793&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1952650" author="xgen-internal-githook" created="Fri, 20 Jul 2018 19:35:32 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Benety Goh&apos;, &apos;email&apos;: &apos;benety@mongodb.com&apos;, &apos;username&apos;: &apos;benety&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-33191&quot; title=&quot;Cache-full hangs on 3.6&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-33191&quot;&gt;&lt;del&gt;SERVER-33191&lt;/del&gt;&lt;/a&gt; add test to fill wiredtiger cache during steady state oplog replication&lt;/p&gt;

&lt;p&gt;(cherry picked from commit 88cea24944cf277759b74b18878a0b4c0d89686e)&lt;br/&gt;
Branch: v4.0&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/b1c9083d0e85f93d842aea6333622093ab0f4279&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/b1c9083d0e85f93d842aea6333622093ab0f4279&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1952649" author="xgen-internal-githook" created="Fri, 20 Jul 2018 19:35:31 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Benety Goh&apos;, &apos;email&apos;: &apos;benety@mongodb.com&apos;, &apos;username&apos;: &apos;benety&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-33191&quot; title=&quot;Cache-full hangs on 3.6&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-33191&quot;&gt;&lt;del&gt;SERVER-33191&lt;/del&gt;&lt;/a&gt; guard mongo test program startup options against duplicate wiredtiger flags&lt;/p&gt;

&lt;p&gt;(cherry picked from commit eaf551c3aa01eb80b48bc7601aa288eed464532c)&lt;br/&gt;
Branch: v4.0&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/9bef62e3f0989e78c9e64c8484e50350522650e6&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/9bef62e3f0989e78c9e64c8484e50350522650e6&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1951645" author="xgen-internal-githook" created="Thu, 19 Jul 2018 19:24:46 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;username&apos;: &apos;benety&apos;, &apos;name&apos;: &apos;Benety Goh&apos;, &apos;email&apos;: &apos;benety@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-33191&quot; title=&quot;Cache-full hangs on 3.6&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-33191&quot;&gt;&lt;del&gt;SERVER-33191&lt;/del&gt;&lt;/a&gt; move wt_cache_full.js from jstests/replsets/ to jstests/noPassthrough/&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/f5035fab356299fe6cffc783dbe9ccfab99fe069&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/f5035fab356299fe6cffc783dbe9ccfab99fe069&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1943247" author="xgen-internal-githook" created="Tue, 10 Jul 2018 15:54:50 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;username&apos;: &apos;benety&apos;, &apos;name&apos;: &apos;Benety Goh&apos;, &apos;email&apos;: &apos;benety@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-33191&quot; title=&quot;Cache-full hangs on 3.6&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-33191&quot;&gt;&lt;del&gt;SERVER-33191&lt;/del&gt;&lt;/a&gt; add test to fill wiredtiger cache during steady state oplog replication&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/88cea24944cf277759b74b18878a0b4c0d89686e&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/88cea24944cf277759b74b18878a0b4c0d89686e&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1943061" author="xgen-internal-githook" created="Tue, 10 Jul 2018 14:36:16 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;username&apos;: &apos;benety&apos;, &apos;name&apos;: &apos;Benety Goh&apos;, &apos;email&apos;: &apos;benety@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-33191&quot; title=&quot;Cache-full hangs on 3.6&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-33191&quot;&gt;&lt;del&gt;SERVER-33191&lt;/del&gt;&lt;/a&gt; guard mongo test program startup options against duplicate wiredtiger flags&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/eaf551c3aa01eb80b48bc7601aa288eed464532c&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/eaf551c3aa01eb80b48bc7601aa288eed464532c&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1906063" author="alexander.gorrod" created="Thu, 31 May 2018 02:34:42 +0000"  >&lt;p&gt;Thanks for the reproducer &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt; I&apos;ve run it and can see the same behavior. We will investigate things we can do to mitigate the issue.&lt;/p&gt;</comment>
                            <comment id="1905187" author="bruce.lucas@10gen.com" created="Wed, 30 May 2018 20:32:07 +0000"  >&lt;p&gt;Attached script repro-10MBx2.sh sometimes gets stuck with the cache full. It&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;starts a repl set using numactl to give each node 2 cpus; 1 GB cache&lt;/li&gt;
	&lt;li&gt;inserts two large (10 MB) documents&lt;/li&gt;
	&lt;li&gt;stops one of the nodes&lt;/li&gt;
	&lt;li&gt;does 1000 updates to each document&lt;/li&gt;
	&lt;li&gt;restarts the node so that it is executing a large batch of replicated updates to catch up; this generates two very large pages and a lot of cache pressure because each replicated batch is pinned in cache in its entirety in 3.6. Checkpoints interval is set to 1 second.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Attached file metrics.2018-05-30T17-31-46Z-00000 shows an instance of the above script running with the normal checkpoint interval. &lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/188039/188039_stuck.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;We get stuck with the cache full at A, coinciding with a checkpoint. (We can tell we&apos;re stuck and haven&apos;t finished catching up because the lag keeps growing.) At A we also see some dhandle sweep activity, but I suspect it&apos;s the checkpoint that matters because increasing the checkpoint interval made the repro more reliable. I don&apos;t know if the checkpoint is integral to the issue, or whether its cpu activity simply opens a window for a race condition, since we only gave the process 2 cpus.&lt;/p&gt;

&lt;p&gt;Stack traces attached. We see two threads applying oplog operations (because we are able to parallelize applying the updates to the two documents), both in __wt_txn_commit, both waiting on a full cache.&lt;/p&gt;</comment>
                            <comment id="1889402" author="bruce.lucas@10gen.com" created="Fri, 11 May 2018 18:15:38 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-34938&quot; title=&quot;Secondary slowdown or hang due to content pinned in cache by single oplog batch&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-34938&quot;&gt;&lt;del&gt;SERVER-34938&lt;/del&gt;&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-34941&quot; title=&quot;Add testing to cover cases where timestamps cause cache pressure&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-34941&quot;&gt;&lt;del&gt;SERVER-34941&lt;/del&gt;&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-34942&quot; title=&quot;Stuck with cache full during oplog replay in initial sync&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-34942&quot;&gt;&lt;del&gt;SERVER-34942&lt;/del&gt;&lt;/a&gt; describe some ways that extreme cache pressure can be generated and in some cases appear to lead to a complete hangs. Those tickets deal with relieving the cause of those instances of cache pressure; I&apos;d like to focus this ticket on why in cases of extreme cache pressure the lookaside mechanism doesn&apos;t prevent us from getting stuck.&lt;/p&gt;</comment>
                            <comment id="1809121" author="michael.cahill" created="Mon, 19 Feb 2018 02:32:48 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt;, you are correct that those stacks indicate that eviction threads are sleeping.  If all four threads are in the same place, then they have each taken a turn at being the server, then given up and gone to sleep before trying again.&lt;/p&gt;

&lt;p&gt;I think we probably have the metrics we need to diagnose what&apos;s happening if we can get FTDC: a combination of the &quot;WT Eviction Efficiency&quot; view plus &quot;eviction state&quot; indicates (a) whether the eviction server is looking for pages at all, (b) what kind of pages the eviction server is trying to queue, (c) how many pages were queued and (d) how many of those pages could be evicted.&lt;/p&gt;

&lt;p&gt;If the cache is full, we would look at what proportion of the cache is dirty &amp;#8211; e.g., one possibility is that the eviction server is stuck looking for dirty pages in a clean cache.  Assuming these issues are not that straightforward, another consideration is whether the cache is clean (dirty % is very low), the eviction state indicates that the eviction server is looking for clean pages but not queuing any.  The only reason not to queue clean leaf pages is if they have history that cannot be evicted.  There was a change in 3.6.0 that was supposed to address this (&lt;a href=&quot;https://jira.mongodb.org/browse/WT-3683&quot; title=&quot;WT tests failing with cache full of clean pages&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-3683&quot;&gt;&lt;del&gt;WT-3683&lt;/del&gt;&lt;/a&gt;), but perhaps it is not effective in these cases.&lt;/p&gt;</comment>
                            <comment id="1808000" author="bruce.lucas@10gen.com" created="Fri, 16 Feb 2018 15:06:03 +0000"  >&lt;p&gt;In all cases seen so far 1-4 of the 4 eviction threads were waiting with this stack:&lt;/p&gt;

&lt;p&gt;    pthread_cond_timedwait@@GLIBC_2.3.2&lt;br/&gt;
    __wt_cond_wait_signal&lt;br/&gt;
    __wt_cond_auto_wait_signal&lt;br/&gt;
    __wt_cond_auto_wait&lt;br/&gt;
    __wt_evict_thread_run&lt;/p&gt;

&lt;p&gt;which I believe is the &lt;a href=&quot;https://github.com/mongodb/mongo/blob/7574817caec09fa1a2ab5ea60f00f766da63f9cd/src/third_party/wiredtiger/src/evict/evict_lru.c#L312&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;eviction server sleeping because it didn&apos;t make progress&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=michael.cahill&quot; class=&quot;user-hover&quot; rel=&quot;michael.cahill&quot;&gt;michael.cahill&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=agorrod&quot; class=&quot;user-hover&quot; rel=&quot;agorrod&quot;&gt;agorrod&lt;/a&gt;, I wonder if we have the metrics in place that will help us understand why it looks like we&apos;re not finding pages to evict, and/or the lookaside table isn&apos;t helping, once &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-32876&quot; title=&quot;Don&amp;#39;t stall ftdc due to WT cache full&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-32876&quot;&gt;&lt;del&gt;SERVER-32876&lt;/del&gt;&lt;/a&gt; is fixed?&lt;/p&gt;

</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                            <outwardlinks description="depends on">
                                        <issuelink>
            <issuekey id="486835">SERVER-32876</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="552075">WT-4105</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="627270">SERVER-37849</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="546808">SERVER-35103</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="574166">SERVER-36221</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="574993">SERVER-36238</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="580291">SERVER-36373</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="552567">WT-4106</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="552570">WT-4107</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="542268">SERVER-34938</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="542299">SERVER-34941</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="542315">SERVER-34942</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="585014">SERVER-36495</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="585017">SERVER-36496</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="188042" name="metrics.2018-05-30T17-31-46Z-00000" size="60110" author="bruce.lucas@mongodb.com" created="Wed, 30 May 2018 20:32:51 +0000"/>
                            <attachment id="188041" name="repro-10MBx2.sh" size="1469" author="bruce.lucas@mongodb.com" created="Wed, 30 May 2018 20:29:45 +0000"/>
                            <attachment id="188040" name="stacks.txt" size="66282" author="bruce.lucas@mongodb.com" created="Wed, 30 May 2018 20:29:45 +0000"/>
                            <attachment id="188039" name="stuck.png" size="169760" author="bruce.lucas@mongodb.com" created="Wed, 30 May 2018 20:29:45 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>15.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="15640"><![CDATA[v4.0]]></customfieldvalue>
    <customfieldvalue key="15141"><![CDATA[v3.6]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[500A000000atJ39IAE]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 19 Feb 2018 02:32:48 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        5 years, 28 weeks, 5 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[<s><a href='https://jira.mongodb.org/browse/SERVER-32876'>SERVER-32876</a></s>, <s><a href='https://jira.mongodb.org/browse/WT-4105'>WT-4105</a></s>]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>PM-1177</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            5 years, 28 weeks, 5 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>alexander.gorrod@mongodb.com</customfieldvalue>
            <customfieldvalue>benety.goh@mongodb.com</customfieldvalue>
            <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>michael.cahill@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htpxaf:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hthf5r:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="2393">Storage NYC 2018-07-16</customfieldvalue>
    <customfieldvalue id="2394">Storage NYC 2018-07-30</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[bruce.lucas@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11861" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>User Summary</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="11856"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htpjf3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>