<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:47:19 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-37886] Remove config server as coordinator crutch from coordinator stepdown targeted tests</title>
                <link>https://jira.mongodb.org/browse/SERVER-37886</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Currently, in order to test coordinator failover, we have an override that causes the router to always use the config server as the coordinator shard for a transaction. Once prepare failover is ready, we should move the coordinator back onto the first shard to be touched by a transaction.&lt;/p&gt;</description>
                <environment></environment>
        <key id="627942">SERVER-37886</key>
            <summary>Remove config server as coordinator crutch from coordinator stepdown targeted tests</summary>
                <type id="3" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14718&amp;avatarType=issuetype">Task</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="jack.mulrow@mongodb.com">Jack Mulrow</assignee>
                                    <reporter username="matthew.saltz@mongodb.com">Matthew Saltz</reporter>
                        <labels>
                            <label>ShardedTxn:DistributedCommit</label>
                    </labels>
                <created>Thu, 1 Nov 2018 22:24:33 +0000</created>
                <updated>Sun, 29 Oct 2023 22:26:56 +0000</updated>
                            <resolved>Wed, 17 Apr 2019 20:48:40 +0000</resolved>
                                                    <fixVersion>4.1.11</fixVersion>
                                    <component>Sharding</component>
                                        <votes>0</votes>
                                    <watches>4</watches>
                                                                                                                <comments>
                            <comment id="2217129" author="xgen-internal-githook" created="Wed, 17 Apr 2019 20:47:41 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;email&apos;: &apos;jack.mulrow@mongodb.com&apos;, &apos;name&apos;: &apos;Jack Mulrow&apos;, &apos;username&apos;: &apos;jsmulrow&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-37886&quot; title=&quot;Remove config server as coordinator crutch from coordinator stepdown targeted tests&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-37886&quot;&gt;&lt;del&gt;SERVER-37886&lt;/del&gt;&lt;/a&gt; Remove config server as coordinator crutch from coordinator stepdown targeted tests&lt;/p&gt;

&lt;p&gt;This reverts commit 7f620154e595d2c1e6c7af79fc62070ced3bb941.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/bb452e17d528733e5e645d814bc237d7a26e6c0b&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/bb452e17d528733e5e645d814bc237d7a26e6c0b&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2214521" author="xgen-internal-githook" created="Tue, 16 Apr 2019 02:17:31 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;email&apos;: &apos;jack.mulrow@mongodb.com&apos;, &apos;name&apos;: &apos;Jack Mulrow&apos;, &apos;username&apos;: &apos;jsmulrow&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-37886&quot; title=&quot;Remove config server as coordinator crutch from coordinator stepdown targeted tests&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-37886&quot;&gt;&lt;del&gt;SERVER-37886&lt;/del&gt;&lt;/a&gt; Make prepareTransaction set Client last OpTime to the greater of the prepare OpTime and system last OpTime if the Client&apos;s last OpTime is less than the prepare OpTime&lt;/p&gt;

&lt;p&gt;This reverts commit bc276668e8d992ff833fdbfd0f4280419d11eda1.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/19b622ebfb42a525f38e278c09f440eb47b12f1e&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/19b622ebfb42a525f38e278c09f440eb47b12f1e&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2184255" author="esha.maharishi@10gen.com" created="Mon, 18 Mar 2019 20:01:48 +0000"  >&lt;p&gt;I also marked this as depends on &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-39036&quot; title=&quot;Stop pinning stable timestamp behind prepared transactions&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-39036&quot;&gt;&lt;del&gt;SERVER-39036&lt;/del&gt;&lt;/a&gt;, because the server fix to the prepare command above (&quot;Make prepareTransaction set Client last OpTime to the greater of the prepare OpTime and system last OpTime if the Client&apos;s last OpTime is less than the prepare OpTime&quot;) exacerbated the existing issue that multiple coordinators for the same session can deadlock with each other on a participant, causing the txn_two_phase_commit_failover.js to fail more frequently than it is now.&lt;/p&gt;</comment>
                            <comment id="2184187" author="esha.maharishi@10gen.com" created="Mon, 18 Mar 2019 19:15:11 +0000"  >&lt;p&gt;Marking as depends on &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-40081&quot; title=&quot;Move session checkout to before command execution&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-40081&quot;&gt;&lt;del&gt;SERVER-40081&lt;/del&gt;&lt;/a&gt;, because currently stepdown can deadlock with a command waiting for writeConcern under a checked out session.&lt;/p&gt;

&lt;p&gt;The deadlock is a result of a lock order inversion between taking the ReplicationCoordinator mutex and checking out a session:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;&lt;a href=&quot;https://github.com/mongodb/mongo/blob/f4375654883489542379abdd627f8e3454f34eaa/src/mongo/db/repl/replication_coordinator_impl.cpp#L1885&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;Stepdown takes the ReplicationCoordinator mutex&lt;/a&gt; first, then while &lt;a href=&quot;https://github.com/mongodb/mongo/blob/f4375654883489542379abdd627f8e3454f34eaa/src/mongo/db/repl/replication_coordinator_impl.cpp#L1998&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;yielding locks for prepared transactions&lt;/a&gt;, tries to &lt;a href=&quot;https://github.com/mongodb/mongo/blob/f4375654883489542379abdd627f8e3454f34eaa/src/mongo/db/kill_sessions_local.cpp#L176&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;check out sessions&lt;/a&gt; (through &lt;a href=&quot;https://github.com/mongodb/mongo/blob/f4375654883489542379abdd627f8e3454f34eaa/src/mongo/db/kill_sessions_local.cpp#L72&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;checkOutSessionForKill&lt;/a&gt;)&lt;/li&gt;
	&lt;li&gt;&lt;a href=&quot;https://github.com/mongodb/mongo/blob/f4375654883489542379abdd627f8e3454f34eaa/src/mongo/db/service_entry_point_common.cpp#L626&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;Commands check out the session&lt;/a&gt; first, then try to take the ReplicationCoordinator mutex as part of &lt;a href=&quot;https://github.com/mongodb/mongo/blob/f4375654883489542379abdd627f8e3454f34eaa/src/mongo/db/service_entry_point_common.cpp#L510&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;waiting for writeConcern&lt;/a&gt; (through &lt;a href=&quot;https://github.com/mongodb/mongo/blob/f4375654883489542379abdd627f8e3454f34eaa/src/mongo/db/repl/replication_coordinator_impl.cpp#L1597&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;awaitReplication&lt;/a&gt;)&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;The deadlock that ensues is either:&lt;/p&gt;

&lt;p&gt;1) Stepdown acquires the ReplicationCoordinator mutex before the command calls ReplicationCoordinator::awaitReplication. The command blocks trying to acquire the ReplicationCoordinator mutex in ReplicationCoordinator::awaitReplication, and the stepdown blocks in checkOutSessionForKill.&lt;/p&gt;

&lt;p&gt;OR&lt;/p&gt;

&lt;p&gt;2) Stepdown acquires the ReplicationCoordinator mutex while the command is in waitForConditionOrInterrupt as part of ReplicationCoordinator::_awaitReplication_inlock. The stepdown successfully marks the command&apos;s OperationContext as killed, but the command blocks trying to acquire the ReplicationCoordinator mutex to wake up from its sleep, and the stepdown blocks in checkOutSessionForKill.&lt;/p&gt;

&lt;p&gt;I originally thought that &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-40069&quot; title=&quot;GlobalLockAcquisitionTracker::getGlobalWriteLocked should return true if the OperationContext it decorates ever held a global write lock&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-40069&quot;&gt;&lt;del&gt;SERVER-40069&lt;/del&gt;&lt;/a&gt; (ensuring the GlobalLockAcquisitionTracker says that an OperationContext acquired a global lock if the OperationContext &lt;b&gt;ever&lt;/b&gt; &lt;b&gt;held&lt;/b&gt; a global lock) would prevent the deadlock, because then the stepdown would mark the OperationContext as killed while not holding the ReplicationCoordinator mutex, through &lt;a href=&quot;https://github.com/mongodb/mongo/blob/f4375654883489542379abdd627f8e3454f34eaa/src/mongo/db/repl/replication_coordinator_impl.cpp#L1969&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;KillOpContainer::startKillOpThread&lt;/a&gt;. However, this would only prevent the second deadlock described above, not the first, since the command would not check for interrupt until after acquiring the ReplicationCoordinator mutex for awaitReplication.&lt;/p&gt;</comment>
                            <comment id="2176055" author="xgen-internal-githook" created="Fri, 8 Mar 2019 18:12:59 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Esha Maharishi&apos;, &apos;username&apos;: &apos;EshaMaharishi&apos;, &apos;email&apos;: &apos;esha.maharishi@mongodb.com&apos;}
&lt;p&gt;Message: Revert &quot;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-37886&quot; title=&quot;Remove config server as coordinator crutch from coordinator stepdown targeted tests&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-37886&quot;&gt;&lt;del&gt;SERVER-37886&lt;/del&gt;&lt;/a&gt; Make prepareTransaction set Client last OpTime to the greater of the prepare OpTime and system last OpTime if the Client&apos;s last OpTime is less than the prepare OpTime&quot;&lt;/p&gt;

&lt;p&gt;This reverts commit 8f3ad3eab4631a393cb4c2bfff69015baf63ebc9.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/bc276668e8d992ff833fdbfd0f4280419d11eda1&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/bc276668e8d992ff833fdbfd0f4280419d11eda1&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2175835" author="xgen-internal-githook" created="Fri, 8 Mar 2019 16:03:05 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Esha Maharishi&apos;, &apos;email&apos;: &apos;esha.maharishi@mongodb.com&apos;, &apos;username&apos;: &apos;EshaMaharishi&apos;}
&lt;p&gt;Message: Revert &quot;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-37886&quot; title=&quot;Remove config server as coordinator crutch from coordinator stepdown targeted tests&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-37886&quot;&gt;&lt;del&gt;SERVER-37886&lt;/del&gt;&lt;/a&gt; Remove config server as coordinator crutch from coordinator stepdown targeted tests&quot;&lt;/p&gt;

&lt;p&gt;This reverts commit a0555e9be29e21c1c822d4bfc860c66b6838a00f.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/7f620154e595d2c1e6c7af79fc62070ced3bb941&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/7f620154e595d2c1e6c7af79fc62070ced3bb941&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2175679" author="xgen-internal-githook" created="Fri, 8 Mar 2019 14:27:46 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Esha Maharishi&apos;, &apos;username&apos;: &apos;EshaMaharishi&apos;, &apos;email&apos;: &apos;esha.maharishi@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-37886&quot; title=&quot;Remove config server as coordinator crutch from coordinator stepdown targeted tests&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-37886&quot;&gt;&lt;del&gt;SERVER-37886&lt;/del&gt;&lt;/a&gt; Remove config server as coordinator crutch from coordinator stepdown targeted tests&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/a0555e9be29e21c1c822d4bfc860c66b6838a00f&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/a0555e9be29e21c1c822d4bfc860c66b6838a00f&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2175678" author="xgen-internal-githook" created="Fri, 8 Mar 2019 14:27:43 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Esha Maharishi&apos;, &apos;username&apos;: &apos;EshaMaharishi&apos;, &apos;email&apos;: &apos;esha.maharishi@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-37886&quot; title=&quot;Remove config server as coordinator crutch from coordinator stepdown targeted tests&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-37886&quot;&gt;&lt;del&gt;SERVER-37886&lt;/del&gt;&lt;/a&gt; Make prepareTransaction set Client last OpTime to the greater of the prepare OpTime and system last OpTime if the Client&apos;s last OpTime is less than the prepare OpTime&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/8f3ad3eab4631a393cb4c2bfff69015baf63ebc9&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/8f3ad3eab4631a393cb4c2bfff69015baf63ebc9&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2084448" author="esha.maharishi@10gen.com" created="Fri, 7 Dec 2018 21:05:51 +0000"  >&lt;p&gt;Actually, using the override that retries the transaction entirely would allow the tests to pass while still testing that the coordinator resumed coordinating commit on stepup.&lt;/p&gt;

&lt;p&gt;I was worried the override would mean the tests would pass even if the coordinator never resumed coordinating any commits, but that&apos;s not true. If the coordinator never resumed coordinating the commit and there were prepared participants, the tests would hang.&lt;/p&gt;</comment>
                            <comment id="2084430" author="esha.maharishi@10gen.com" created="Fri, 7 Dec 2018 20:50:11 +0000"  >&lt;p&gt;Since participants abort unprepared transactions on stepdown, we will never be able to do &lt;b&gt;passthrough&lt;/b&gt;&#160;testing of coordinator failover unless the coordinator is not a participant (the tests running in the passthrough may expect the transaction to commit).&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                            <outwardlinks description="depends on">
                                        <issuelink>
            <issuekey id="565188">SERVER-35872</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="673272">SERVER-39036</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="715038">SERVER-40081</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is depended on by">
                                        <issuelink>
            <issuekey id="719325">SERVER-40220</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>10.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 7 Dec 2018 20:50:11 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        4 years, 43 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[<s><a href='https://jira.mongodb.org/browse/SERVER-35872'>SERVER-35872</a></s>, <s><a href='https://jira.mongodb.org/browse/SERVER-40081'>SERVER-40081</a></s>, <s><a href='https://jira.mongodb.org/browse/SERVER-39036'>SERVER-39036</a></s>]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                            <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>PM-564</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            4 years, 43 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_16465" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Linked BF Score</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>11.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>esha.maharishi@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>jack.mulrow@mongodb.com</customfieldvalue>
            <customfieldvalue>matthew.saltz@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hubvlr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hufpmn:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="2824">Sharding 2019-03-11</customfieldvalue>
    <customfieldvalue id="2864">Sharding 2019-04-22</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hubhv3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>