<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:38:19 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-34938] Secondary slowdown or hang due to content pinned in cache by single oplog batch</title>
                <link>https://jira.mongodb.org/browse/SERVER-34938</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;We only advance the oldest timestamp at oplog batch boundaries. This means that all dirty content generated by the application of the operations in a single batch will be pinned in cache. If the batch is large enough and the operations are heavy enough this dirty content can exceed eviction_dirty_trigger (default 20% of cache) and the rate of applying operations will become dramatically slower because it has to wait for the dirty data to be reduced below the threshold.&lt;/p&gt;

&lt;p&gt;This can be triggered by a momentary slowdown on a secondary causing it to lag momentarily, so the next batch it processes will be unusually large, causing it to exceed 20% dirty cache. This will cause it to lag even further, so the next batch will be even larger, and so on. In extreme cases the node can become completely stuck due to full cache preventing a batch from completing and unpinning the data that is keeping the cache full.&lt;/p&gt;

&lt;p&gt;This can also occur if a secondary is offline for maintenance; when it comes back online and begins to catch up, it will be processing large batches that risk exceeding the dirty trigger threshold, so it may apply operations at a much slower rate than a secondary that is keeping up and processing operations in small batches.&lt;/p&gt;</description>
                <environment></environment>
        <key id="542268">SERVER-34938</key>
            <summary>Secondary slowdown or hang due to content pinned in cache by single oplog batch</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="m.maher@mongodb.com">Moustafa Maher</assignee>
                                    <reporter username="bruce.lucas@mongodb.com">Bruce Lucas</reporter>
                        <labels>
                            <label>dmd-perf</label>
                            <label>former-quick-wins</label>
                    </labels>
                <created>Thu, 10 May 2018 19:50:04 +0000</created>
                <updated>Sun, 29 Oct 2023 22:31:54 +0000</updated>
                            <resolved>Thu, 1 Apr 2021 23:49:08 +0000</resolved>
                                    <version>3.6.3</version>
                    <version>3.6.4</version>
                                    <fixVersion>5.0.0-rc0</fixVersion>
                    <fixVersion>4.2.16</fixVersion>
                    <fixVersion>4.0.27</fixVersion>
                    <fixVersion>4.4.9</fixVersion>
                                    <component>Replication</component>
                                        <votes>6</votes>
                                    <watches>78</watches>
                                                                                                                <comments>
                            <comment id="4030285" author="xgen-internal-githook" created="Mon, 30 Aug 2021 21:37:29 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Moustafa Maher&apos;, &apos;email&apos;: &apos;m.maher@10gen.com&apos;, &apos;username&apos;: &apos;moustafamaher&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-34938&quot; title=&quot;Secondary slowdown or hang due to content pinned in cache by single oplog batch&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-34938&quot;&gt;&lt;del&gt;SERVER-34938&lt;/del&gt;&lt;/a&gt; Secondary slowdown or hang due to content pinned in cache by single oplog batch&lt;br/&gt;
Branch: v4.0&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/d47b151b55f286546e7c7c98888ae0577856ca20&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/d47b151b55f286546e7c7c98888ae0577856ca20&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="4029923" author="xgen-internal-githook" created="Mon, 30 Aug 2021 19:11:38 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Moustafa Maher&apos;, &apos;email&apos;: &apos;m.maher@10gen.com&apos;, &apos;username&apos;: &apos;moustafamaher&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-34938&quot; title=&quot;Secondary slowdown or hang due to content pinned in cache by single oplog batch&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-34938&quot;&gt;&lt;del&gt;SERVER-34938&lt;/del&gt;&lt;/a&gt; Secondary slowdown or hang due to content pinned in cache by single oplog batch&lt;br/&gt;
Branch: v4.2&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/8a5c1f52a77bc92320c90ecbe5424e841874296e&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/8a5c1f52a77bc92320c90ecbe5424e841874296e&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3981916" author="xgen-internal-githook" created="Wed, 4 Aug 2021 23:13:05 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Moustafa Maher&apos;, &apos;email&apos;: &apos;m.maher@10gen.com&apos;, &apos;username&apos;: &apos;moustafamaher&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-34938&quot; title=&quot;Secondary slowdown or hang due to content pinned in cache by single oplog batch&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-34938&quot;&gt;&lt;del&gt;SERVER-34938&lt;/del&gt;&lt;/a&gt; Secondary slowdown or hang due to content pinned in cache by single oplog batch&lt;br/&gt;
Branch: v4.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/ac3ff8214bf42119945d49d183d5410bd81a2913&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/ac3ff8214bf42119945d49d183d5410bd81a2913&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3697687" author="xgen-internal-githook" created="Thu, 1 Apr 2021 23:01:27 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Moustafa Maher&apos;, &apos;email&apos;: &apos;m.maher@10gen.com&apos;, &apos;username&apos;: &apos;moustafamaher&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-34938&quot; title=&quot;Secondary slowdown or hang due to content pinned in cache by single oplog batch&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-34938&quot;&gt;&lt;del&gt;SERVER-34938&lt;/del&gt;&lt;/a&gt; Secondary slowdown or hang due to content pinned in cache by single oplog batch&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/39e9990a2fe1b72e5d776a86651469894d0d72af&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/39e9990a2fe1b72e5d776a86651469894d0d72af&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3574519" author="alexander.gorrod" created="Wed, 20 Jan 2021 20:58:15 +0000"  >&lt;blockquote&gt;&lt;p&gt;Is it correct to assume that Durable History will make this problem go away?&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;It is unlikely that durable history will help in this case. The transaction(s) being used to apply the batch of oplog entries are pinning the cache, and even with durable history, WiredTiger can&apos;t evict content related to active transactions.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;i.e: I don&apos;t believe the issue occurs with a combination of committed and uncommitted data that can&apos;t be evicted. It is that there is too much uncommitted data outright.&lt;/p&gt;</comment>
                            <comment id="2088282" author="bruce.lucas@10gen.com" created="Wed, 12 Dec 2018 12:58:22 +0000"  >&lt;p&gt;Attached a repro, batchsize-1.{sh,js}, that&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;starts a workload that generates a large amount of dirty cache content per replicated op&lt;/li&gt;
	&lt;li&gt;after 60 seconds, we slow one of the secondaries for 30 seconds by stopping it for 0.8 seconds out of every second, generating some lag&lt;/li&gt;
	&lt;li&gt;after that 30 second slowdown, that secondary is stuck in a low-performance state permanently or for an extended time so it&apos;s not able to catch up&lt;/li&gt;
&lt;/ul&gt;
</comment>
                            <comment id="1949023" author="bruce.lucas@10gen.com" created="Tue, 17 Jul 2018 14:09:40 +0000"  >&lt;p&gt;Note that a side effect of this issue can be unbounded lookaside table growth, which results in unbounded growth in WiredTigerLAS.wt.&lt;/p&gt;</comment>
                            <comment id="1905231" author="bruce.lucas@10gen.com" created="Wed, 30 May 2018 21:02:04 +0000"  >&lt;p&gt;The reproducer on &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-33191&quot; title=&quot;Cache-full hangs on 3.6&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-33191&quot;&gt;&lt;del&gt;SERVER-33191&lt;/del&gt;&lt;/a&gt; also reproduces this issue, as the hang that it reproduces is triggered by the cache pressure caused by this issue.&lt;/p&gt;</comment>
                            <comment id="1899926" author="spencer" created="Wed, 23 May 2018 19:06:54 +0000"  >&lt;p&gt;FYI, there is a setParameter currently, replBatchLimitOperations, which controls the max number of oplog entries in a singe replication batch.  If users are hitting this issue, they can consider turning that value down from its default of 50,000 to produce smaller secondary batches.&lt;br/&gt;
Note however that this is a limit on number of documents in a batch, which may not be a great proxy for size of the batch in terms of MB of oplog entries or in terms of number of pages touched by applying the batch, therefore it&apos;s difficult to directly map how changes to this parameter will affect changes to cache pressure associated with applying batches.&lt;/p&gt;</comment>
                            <comment id="1895150" author="bruce.lucas@10gen.com" created="Fri, 18 May 2018 14:26:37 +0000"  >&lt;p&gt;We need to consider the impact of replicated index builds on this issue. The index build will generate some pinned cache content during the application of a batch, but won&apos;t be included in any accounting of the direct impact of the batch.&lt;/p&gt;</comment>
                            <comment id="1889545" author="bruce.lucas@10gen.com" created="Fri, 11 May 2018 20:16:10 +0000"  >&lt;p&gt;Summarizing some offline conversations with &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=schwerin&quot; class=&quot;user-hover&quot; rel=&quot;schwerin&quot;&gt;schwerin&lt;/a&gt; and &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=michael.cahill&quot; class=&quot;user-hover&quot; rel=&quot;michael.cahill&quot;&gt;michael.cahill&lt;/a&gt;&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;the point of the markers above was to avoid adding a field to every oplog entry, assuming that may be difficult. If it&apos;s possible to add a field to every oplog entry that&apos;s simpler.&lt;/li&gt;
	&lt;li&gt;any measure used to adaptively adjust the size of the batches on the primary should be equally applicable to all secondaries. Something based on actual page sizes might not satisfy that as such details could differ on the secondaries.&lt;/li&gt;
	&lt;li&gt;a simple measure based on number of pages touched might be a sufficient rough approximation&lt;/li&gt;
	&lt;li&gt;an even simpler approach of just limiting the maximum batch size might be sufficient, and more suitable for backport&lt;/li&gt;
&lt;/ul&gt;
</comment>
                            <comment id="1888980" author="bruce.lucas@10gen.com" created="Fri, 11 May 2018 12:14:53 +0000"  >&lt;p&gt;I&apos;m not sure that&apos;s sufficient. In some customer workloads we&apos;ve seen a single large batch delay things for a very long time, particularly in the situation where we&apos;re catching up from a large lag.&lt;/p&gt;

&lt;p&gt;How about something like the following:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;estimate cache pressure or potential cache pressure created by each op on the primary. This could be done maybe by
	&lt;ul&gt;
		&lt;li&gt;having the storage engine to give an accounting or estimate of pinned content or potential pinned content (after reconciliation) for each op&lt;/li&gt;
		&lt;li&gt;using number of pages touched (maybe number of inserts and deletes) as a rough proxy&lt;/li&gt;
		&lt;li&gt;every so many ops querying the storage engine for statistics related to pinned content&lt;/li&gt;
	&lt;/ul&gt;
	&lt;/li&gt;
	&lt;li&gt;every time this estimate crosses some fixed boundary, e.g. 5% of cache size, insert a marker in the oplog&lt;/li&gt;
	&lt;li&gt;on the secondaries never include more than one such marker in a batch (allowing one ensures we don&apos;t get tiny batches near boundaries, if that&apos;s important)&lt;/li&gt;
&lt;/ul&gt;

</comment>
                            <comment id="1888766" author="schwerin" created="Fri, 11 May 2018 00:42:34 +0000"  >&lt;p&gt;We can easily reduce the maximum batch size as measured in count of oplog entries or bytes of oplog entries, but that may be a poor proxy for cache pressure caused by a batch.&lt;/p&gt;

&lt;p&gt;Would it be enough to reduce the batch size for subsequent batches during periods of high cache pressure?&lt;/p&gt;</comment>
                            <comment id="1888594" author="bruce.lucas@10gen.com" created="Thu, 10 May 2018 20:42:30 +0000"  >&lt;p&gt;Note I&apos;m not sure whether the &quot;node can become completely stuck due to full cache&quot; part of the problem description is a necessary consequence of this issue, or whether the lookaside mechanism is meant to prevent that and limit the consequences of this issue to an extreme slowdown. If so there may be an additional ticket (but fixing lookaside won&apos;t fix the extreme performance consequences of this issue).&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                                                <inwardlinks description="is depended on by">
                                        <issuelink>
            <issuekey id="567709">SERVER-35958</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10320">
                    <name>Documented</name>
                                                                <inwardlinks description="is documented by">
                                        <issuelink>
            <issuekey id="2073774">DOCS-15433</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                                                <inwardlinks description="is duplicated by">
                                        <issuelink>
            <issuekey id="552745">SERVER-35339</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10520">
                    <name>Problem/Incident</name>
                                            <outwardlinks description="causes">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="627270">SERVER-37849</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="494491">SERVER-33191</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="542299">SERVER-34941</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="542315">SERVER-34942</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="546808">SERVER-35103</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="548286">SERVER-35191</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="585014">SERVER-36495</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="585017">SERVER-36496</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="554518">SERVER-35405</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="203421" name="batchsize-1.js" size="463" author="bruce.lucas@mongodb.com" created="Wed, 12 Dec 2018 12:53:19 +0000"/>
                            <attachment id="203420" name="batchsize-1.sh" size="1900" author="bruce.lucas@mongodb.com" created="Wed, 12 Dec 2018 12:52:55 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>14.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="21266"><![CDATA[v4.9]]></customfieldvalue>
    <customfieldvalue key="18953"><![CDATA[v4.4]]></customfieldvalue>
    <customfieldvalue key="16775"><![CDATA[v4.2]]></customfieldvalue>
    <customfieldvalue key="15640"><![CDATA[v4.0]]></customfieldvalue>
    <customfieldvalue key="15141"><![CDATA[v3.6]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10011"><![CDATA[Minor Change]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[500A000000asfGgIAI, 500A000000aqs0yIAA, 500A000000aPzntIAC, 500A000000atJ39IAE, 500A000000b8Kz3IAE, 500A000000bViiIIAS, 500A000000bROXGIA4, 500A000000cESOpIAO, 500A000000cFIiZIAW, 500A000000cPngIIAS, 500A000000cddkCIAQ, 5002K00000cxSfhQAE, 5002K00000d1590QAA, 5002K00000d6TLxQAM, 5002K00000dXh4LQAS, 5002K00000e6PrnQAE, 5002K00000dYcBzQAK, 5002K00000e9sGtQAI, 5002K00000fFl4NQAS, 5002K00000fGTb1QAG, 5002K00000f1ElmQAE, 5002K00000f2ZLJQA2, 5002K00000g480IQAQ, 5002K00000gnpxoQAA, 5002K00000gnnFMQAY, 5002K00000hwADPQA2, 5002K00000hRkqrQAC, 5002K00000iMiatQAC, 5002K00000iOK4iQAG, 5002K00000j2Qf2QAE, 5002K00000iQLYnQAO, 5002K00000fFlahQAC, 5002K00000ejqy5QAA, 5002K00000dMLEMQA4, 5002K00000dZ0TBQA0, 5002K00000jcisnQAA, 5002K00000kEhFLQA0, 5002K00000kEvlIQAS, 5002K00000kFvkuQAC, 5002K00000kqv6RQAQ, 5002K00000lmOtYQAU, 5002K00000huqr3QAA, 5002K00000lnoAYQAY, 5002K00000msPjbQAE, 5002K00000msbRiQAI, 5002K00000nmKabQAE, 5002K00000nmbbMQAQ, 5002K00000nmkXDQAY, 5002K00000nmpEsQAI, 5002K00000ocb1oQAA, 5002K00000ocJUKQA2, 5002K00000ocoCGQAY, 5002K00000ocY9iQAE, 5002K00000pD2ooQAC, 5002K00000pEzdZQAS, 5002K00000pFWfoQAG, 5002K00000pmxKfQAI, 5002K00000qcuj2QAA, 5002K00000r33MKQAY, 5002K00000r3MtUQAU, 5002K00000r3jMQQAY, 5002K00000r4KB7QAM, 5002K00000s0haCQAQ, 5002K00000sb1xyQAA, 5002K00000saiITQAY, 5002K00000sbZWPQA2, 5002K00000tR7i9QAC, 5002K00000uFMYRQA4, 5002K00000uGF3jQAG, 5002K00000uysDzQAI, 5002K00000vfGmXQAU, 5002K00000vgRK7QAM, 5002K00000xCkBFQA0, 5006R00001mtxO9QAI, 5006R00001ooTFWQA2]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 11 May 2018 00:42:34 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        2 years, 23 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_17052" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Downstream Changes Summary</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Document that maxNumberOfThreads is now capped with number of available cores.</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16942"><![CDATA[Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>PM-1867</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            2 years, 23 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>alexander.gorrod@mongodb.com</customfieldvalue>
            <customfieldvalue>schwerin@mongodb.com</customfieldvalue>
            <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>m.maher@mongodb.com</customfieldvalue>
            <customfieldvalue>spencer@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htxrzz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr4k8f:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="4756">Repl 2021-03-08</customfieldvalue>
    <customfieldvalue id="4798">Repl 2021-04-05</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_17051" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Teams Impacted</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16944"><![CDATA[Docs]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[kelsey.schubert@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htxe9b:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>