<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:46:02 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-58988] Avoid sync source selection cycle during primary catchup.</title>
                <link>https://jira.mongodb.org/browse/SERVER-58988</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;During primary catchup, the primary does not necessarily sync from the most up-to-date node, we assume its sync source will sync from a more up-to-date node. However in some cases, the primary that is doing catchup can end up in a situation that it is syncing from a node who in turn selects this primary as sync source. This forms a sync source selection cycle and if this sync source node is not up-to-date in primary&apos;s view, then the primary can never be able to catchup to the &lt;a href=&quot;https://github.com/mongodb/mongo/blob/ce2afc61f8326797a0bcfae9e2bf4b145e430017/src/mongo/db/repl/replication_coordinator_impl.cpp#L4408&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;target opTime&lt;/a&gt;, which causes a long unavailability window.&lt;/p&gt;

&lt;p&gt;If this happens, we rely on catchupTimeout (by default infinite), catchup takeover (by default 30s) or manual abort to bail it out, but the long unavailability window is undesirable.&lt;/p&gt;</description>
                <environment></environment>
        <key id="1837103">SERVER-58988</key>
            <summary>Avoid sync source selection cycle during primary catchup.</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="wenbin.zhu@mongodb.com">Wenbin Zhu</assignee>
                                    <reporter username="wenbin.zhu@mongodb.com">Wenbin Zhu</reporter>
                        <labels>
                    </labels>
                <created>Fri, 30 Jul 2021 22:55:27 +0000</created>
                <updated>Sun, 29 Oct 2023 21:50:07 +0000</updated>
                            <resolved>Thu, 2 Sep 2021 03:03:40 +0000</resolved>
                                                    <fixVersion>4.2.18</fixVersion>
                    <fixVersion>4.4.10</fixVersion>
                    <fixVersion>5.0.4</fixVersion>
                    <fixVersion>5.1.0-rc0</fixVersion>
                                    <component>Replication</component>
                                        <votes>0</votes>
                                    <watches>8</watches>
                                                                                                                <comments>
                            <comment id="4107826" author="JIRAUSER1259052" created="Wed, 6 Oct 2021 18:37:23 +0000"  >&lt;p&gt;Updating the fixversion since branching activities occurred yesterday. This ticket will be in rc0 when it&#8217;s been triggered. For more active release information, please keep an eye on #server-release. Thank you!&lt;/p&gt;</comment>
                            <comment id="4099828" author="xgen-internal-githook" created="Mon, 4 Oct 2021 15:49:27 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Wenbin Zhu&apos;, &apos;email&apos;: &apos;wenbin.zhu@mongodb.com&apos;, &apos;username&apos;: &apos;WenbinZhu&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-58988&quot; title=&quot;Avoid sync source selection cycle during primary catchup.&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-58988&quot;&gt;&lt;del&gt;SERVER-58988&lt;/del&gt;&lt;/a&gt; Avoid sync source selection cycle during primary catchup.&lt;/p&gt;

&lt;p&gt;(cherry picked from commit b46acdbba8ec51810b6f402dbe18ed7ea98fd13d)&lt;br/&gt;
Branch: v4.2&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/d9dba1a12f4e1257b711c5e7859de7d7680d5bc6&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/d9dba1a12f4e1257b711c5e7859de7d7680d5bc6&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="4097285" author="xgen-internal-githook" created="Fri, 1 Oct 2021 18:39:02 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Wenbin Zhu&apos;, &apos;email&apos;: &apos;wenbin.zhu@mongodb.com&apos;, &apos;username&apos;: &apos;WenbinZhu&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-58988&quot; title=&quot;Avoid sync source selection cycle during primary catchup.&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-58988&quot;&gt;&lt;del&gt;SERVER-58988&lt;/del&gt;&lt;/a&gt; Avoid sync source selection cycle during primary catchup.&lt;/p&gt;

&lt;p&gt;(cherry picked from commit b46acdbba8ec51810b6f402dbe18ed7ea98fd13d)&lt;br/&gt;
Branch: v4.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/eea1bcb22a3d897cccd8f923350df380db203732&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/eea1bcb22a3d897cccd8f923350df380db203732&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="4097196" author="xgen-internal-githook" created="Fri, 1 Oct 2021 18:07:25 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Wenbin Zhu&apos;, &apos;email&apos;: &apos;wenbin.zhu@mongodb.com&apos;, &apos;username&apos;: &apos;WenbinZhu&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-58988&quot; title=&quot;Avoid sync source selection cycle during primary catchup.&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-58988&quot;&gt;&lt;del&gt;SERVER-58988&lt;/del&gt;&lt;/a&gt; Avoid sync source selection cycle during primary catchup.&lt;/p&gt;

&lt;p&gt;(cherry picked from commit b46acdbba8ec51810b6f402dbe18ed7ea98fd13d)&lt;br/&gt;
Branch: v5.0&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/2d64fa08148b302eae3565123a1752bb895bfbaf&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/2d64fa08148b302eae3565123a1752bb895bfbaf&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="4035557" author="xgen-internal-githook" created="Thu, 2 Sep 2021 02:54:02 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Wenbin Zhu&apos;, &apos;email&apos;: &apos;wenbin.zhu@mongodb.com&apos;, &apos;username&apos;: &apos;WenbinZhu&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-58988&quot; title=&quot;Avoid sync source selection cycle during primary catchup.&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-58988&quot;&gt;&lt;del&gt;SERVER-58988&lt;/del&gt;&lt;/a&gt; Avoid sync source selection cycle during primary catchup.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/b46acdbba8ec51810b6f402dbe18ed7ea98fd13d&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/b46acdbba8ec51810b6f402dbe18ed7ea98fd13d&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3978736" author="siyuan.zhou@10gen.com" created="Tue, 3 Aug 2021 18:22:29 +0000"  >&lt;p&gt;I don&apos;t think we modeled sync source selection in the TLA+ spec. I remembered we wanted to extend the TLA+ spec for that when Samy and Xuerui were working on the sync source selection. Happy to brainstorm if you need.&lt;/p&gt;</comment>
                            <comment id="3976640" author="JIRAUSER1258790" created="Mon, 2 Aug 2021 20:47:06 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=judah.schvimer&quot; class=&quot;user-hover&quot; rel=&quot;judah.schvimer&quot;&gt;judah.schvimer&lt;/a&gt;&#160;after a second thought, I think this might not be a big problem. The scenario I was describing is that say n2 is the primary elect, n0 was old primary and is up-to-date, n1 is syncing from n0, n2 is syncing from n1 and now n1,n2 are equally up-to-date. Now if there is a partition between n0 and n1, n1 needs to change sync source but will not sync from n2 because chaining is enabled and n2 is not ahead of it, so n1 will clear its sync source. When &lt;a href=&quot;https://github.com/mongodb/mongo/blob/a33a04b6186ea5b56c1c9228ed19c41061f80749/src/mongo/db/repl/topology_coordinator.cpp#L3068-L3078&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;n2 sees n1 cleared its sync source, n2 will also select a new sync source&lt;/a&gt;, which will be n0 and so we won&apos;t be stuck, it&apos;s just that n2 will spend a bit more time to converge.&lt;/p&gt;</comment>
                            <comment id="3976493" author="judah.schvimer" created="Mon, 2 Aug 2021 19:59:48 +0000"  >&lt;blockquote&gt;
&lt;p&gt;if there is a partition between n0 and n1, and primary selects n1 as its sync source, we can still get stuck&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=wenbin.zhu&quot; class=&quot;user-hover&quot; rel=&quot;wenbin.zhu&quot;&gt;wenbin.zhu&lt;/a&gt;, I&apos;m not following this scenario. I&apos;d expect the above TLA+ spec to catch this case. Can you please elaborate on this scenario?&lt;/p&gt;</comment>
                            <comment id="3976320" author="judah.schvimer" created="Mon, 2 Aug 2021 19:12:09 +0000"  >&lt;p&gt;I think as part of this ticket, we should model the bug and the fix in the &lt;a href=&quot;https://github.com/10gen/mongo/blob/8fcfd9dd7015a054887fa6ea9ecb3e1a9c6556c0/src/mongo/db/repl/tla_plus/RaftMongo/RaftMongo.tla&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;RaftMongo.tla&lt;/a&gt; spec. CC &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=siyuan.zhou&quot; class=&quot;user-hover&quot; rel=&quot;siyuan.zhou&quot;&gt;siyuan.zhou&lt;/a&gt; and &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=jesse&quot; class=&quot;user-hover&quot; rel=&quot;jesse&quot;&gt;jesse&lt;/a&gt; who may be interested.&lt;/p&gt;</comment>
                            <comment id="3976287" author="JIRAUSER1258790" created="Mon, 2 Aug 2021 19:01:12 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=judah.schvimer&quot; class=&quot;user-hover&quot; rel=&quot;judah.schvimer&quot;&gt;judah.schvimer&lt;/a&gt;&#160;&lt;/p&gt;
&lt;blockquote&gt;&lt;p&gt;When chaining is disabled, why is a node allowed to sync from a primary that&apos;s not ahead of it?&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;When chaining is disabled, we call `_choosePrimaryAsSyncSource` where we &lt;a href=&quot;https://github.com/mongodb/mongo/blob/93fd56bef8e72ab260a5338deeaf463f5b871099/src/mongo/db/repl/topology_coordinator.cpp#L634-L643&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;allow choosing a primary that is equally up-to-date&lt;/a&gt;, which seems make sense because we can only sync from primary, but we can rethink about this decision. And I think you&apos;re right that this problem is exclusively when chaining is disable, because when chaining is enabled, we &lt;a href=&quot;https://github.com/mongodb/mongo/blob/93fd56bef8e72ab260a5338deeaf463f5b871099/src/mongo/db/repl/topology_coordinator.cpp#L508-L517&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;don&apos;t allow choosing a sync source that is equally up-to-date&lt;/a&gt;. But I think there is one case that needs to be considered when chaining is enabled: if there is a partition between n0 and n1, and primary selects n1 as its sync source, we can still get stuck, but if we choose n0 in the first place (with the first approach), we can avoid this case, that&apos;s one of the reason that we initially want to combine both approaches as a better solution.&lt;/p&gt;</comment>
                            <comment id="3976214" author="judah.schvimer" created="Mon, 2 Aug 2021 18:32:53 +0000"  >&lt;p&gt;I&apos;d be pretty nervous about the first solution. This problem is exclusively for when chaining is disabled, correct? If so, I&apos;m inclined to make the fix only for when chaining is disabled. When chaining is disabled, why is a node allowed to sync from a primary that&apos;s not ahead of it? Nodes syncing from nodes not strictly ahead of them seems prone to cycles.&lt;/p&gt;</comment>
                            <comment id="3976149" author="JIRAUSER1258790" created="Mon, 2 Aug 2021 18:04:35 +0000"  >&lt;p&gt;Hey&#160;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=pavithra.vetriselvan&quot; class=&quot;user-hover&quot; rel=&quot;pavithra.vetriselvan&quot;&gt;pavithra.vetriselvan&lt;/a&gt;, currently when checking&#160;&lt;tt&gt;shouldChangeSyncSource,&lt;/tt&gt;&#160;we know the &lt;a href=&quot;https://github.com/mongodb/mongo/blob/ce2afc61f8326797a0bcfae9e2bf4b145e430017/src/mongo/db/repl/topology_coordinator.cpp#L3068&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;&lt;tt&gt;syncSourceIndex&lt;/tt&gt;&lt;/a&gt; from &lt;tt&gt;OplogQueryMetadata&lt;/tt&gt;. This can serve the purpose of knowing which node is syncing from us, although using&#160;&lt;tt&gt;syncSourceIndex&lt;/tt&gt; is not 100% correct, since the index might com from a different config (similar to&#160;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-58787&quot; title=&quot;Incorrect use of candidateIndex from a previous config during vote processing.&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-58787&quot;&gt;&lt;del&gt;SERVER-58787&lt;/del&gt;&lt;/a&gt;). But because add/remove node is not very frequent and we can probably withstanding making some incorrect shouldChangeSyncSource decisions, so it might be fine to use it, though the better way is to add the information of the sync source&apos;s host or nodeId instead of index.&lt;/p&gt;</comment>
                            <comment id="3975471" author="pavithra.vetriselvan" created="Mon, 2 Aug 2021 14:30:19 +0000"  >&lt;p&gt;The second solution feels a little more intuitive to me, but do we currently have a way to track which nodes are syncing from us? Would we have to add that mechanism?&lt;/p&gt;</comment>
                            <comment id="3973998" author="JIRAUSER1258790" created="Fri, 30 Jul 2021 23:35:22 +0000"  >&lt;p&gt;1) How exactly can this happen:&lt;/p&gt;

&lt;p&gt;Say we have 3 nodes and &lt;b&gt;chaining is disabled&lt;/b&gt;. n0 is the previous primary with is has newer lastApplied opTime than the other two nodes. n2 was elected to be the new primary and entered catchup mode with vote from n1. Now n1 is syncing from n0, and synced some new entries from n0 but not the newest yet. The primary n2 starts to select its sync source for catchup and because in this case we choose sync source &lt;a href=&quot;https://github.com/mongodb/mongo/blob/ce2afc61f8326797a0bcfae9e2bf4b145e430017/src/mongo/db/repl/replication_coordinator_impl.cpp#L4850&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;based on ping latency&lt;/a&gt;, n2 could choose n1 as its sync source. Then n2 quickly caught up to n1 and they become equal. At the end of each opLogFetcher&apos;s successful batch, we check to see if we &lt;a href=&quot;https://github.com/mongodb/mongo/blob/ce2afc61f8326797a0bcfae9e2bf4b145e430017/src/mongo/db/repl/topology_coordinator.cpp#L3002&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;should keep syncing from the current sync source,&lt;/a&gt;&#160;and let&apos;s say at this point n1 has not changed its sync source to n2 (due to n2 became primary, n1 will eventually sync from n2 &lt;a href=&quot;https://github.com/mongodb/mongo/blob/ce2afc61f8326797a0bcfae9e2bf4b145e430017/src/mongo/db/repl/topology_coordinator.cpp#L3059&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;because chaining is disabled&lt;/a&gt;). Since n1 still has a sync source, n2 keep thinking that n1 is a &lt;a href=&quot;https://github.com/mongodb/mongo/blob/ce2afc61f8326797a0bcfae9e2bf4b145e430017/src/mongo/db/repl/topology_coordinator.cpp#L3068-L3078&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;valid sync source for it&lt;/a&gt;. Now n1 realizes that n2 becomes primary so it will &lt;a href=&quot;https://github.com/mongodb/mongo/blob/ce2afc61f8326797a0bcfae9e2bf4b145e430017/src/mongo/db/repl/topology_coordinator.cpp#L3056-L3064&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;choose n2 as its new sync source&lt;/a&gt;. Now the sync source selection cycle is formed and the problem is this cycle is not just transient, both n2 and n1 will not change their sync source after that. For n2 even though once a while the fetcher timeout and returns an empty batch which allow n2 to re-check if sync source is valid, it always thinks &lt;a href=&quot;https://github.com/mongodb/mongo/blob/ce2afc61f8326797a0bcfae9e2bf4b145e430017/src/mongo/db/repl/topology_coordinator.cpp#L3068-L3078&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;n1 is its valid sync source&lt;/a&gt; because n1 has its own sync source, and for n1, since chaining is disabled, n2 is its only valid sync source, and we don&apos;t require a sync source to be ahead of us if it is a primary so n1 won&apos;t change its sync source either and the problem persists.&lt;/p&gt;

&lt;p&gt;2) How can we improve to avoid it:&lt;/p&gt;

&lt;p&gt;For now, we think there are two things that can be done to avoid the sync source selection cycle. First when selecting sync source when we&apos;re in primary catchup, instead of choosing based on ping latency, we can sort the candidates based on whoever is newer in oplog and select the most up-to-date node that is alive. Another thing to do is when checking &lt;tt&gt;shouldChangeSyncSource&lt;/tt&gt;&#160;after each batch, we return true if we are primary and already caught up to the sync source, but the sync source is also syncing from us, thus breaking the cycle.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                                                <inwardlinks description="is depended on by">
                                        <issuelink>
            <issuekey id="1865694">SERVER-59732</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="1759532">SERVER-57262</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>14.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="21777"><![CDATA[v5.0]]></customfieldvalue>
    <customfieldvalue key="18953"><![CDATA[v4.4]]></customfieldvalue>
    <customfieldvalue key="16775"><![CDATA[v4.2]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 2 Aug 2021 14:30:19 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        2 years, 18 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            2 years, 18 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>judah.schvimer@mongodb.com</customfieldvalue>
            <customfieldvalue>pavithra.vetriselvan@mongodb.com</customfieldvalue>
            <customfieldvalue>siyuan.zhou@mongodb.com</customfieldvalue>
            <customfieldvalue>vivian.ge@mongodb.com</customfieldvalue>
            <customfieldvalue>wenbin.zhu@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hzvd2f:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hyu0jj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="5141">Repl 2021-08-23</customfieldvalue>
    <customfieldvalue id="5142">Repl 2021-09-06</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hzuzbj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>