<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:11:19 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-46381] Test concurrent reconfig and stepdown</title>
                <link>https://jira.mongodb.org/browse/SERVER-46381</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;The safe reconfig protocol ensures only one reconfig can happen at a time (on a given node), and reconfig cancels elections. However, a node that has learned about a new config via heartbeat and is beginning to process it could suddenly start to step down. Let&apos;s investigate whether there are concurrency issues in these scenarios.&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;node has started to step down and suddenly learns about a new config via heartbeat&lt;/li&gt;
	&lt;li&gt;a node has learned about a new config via heartbeat, but suddenly starts to step down&lt;/li&gt;
	&lt;li&gt;a node has started to step down and suddenly a replSetReconfig command arrives from the user&lt;/li&gt;
	&lt;li&gt;a replSetReconfig command arrives from the user and suddenly the node starts to step down&lt;/li&gt;
&lt;/ol&gt;
</description>
                <environment></environment>
        <key id="1182491">SERVER-46381</key>
            <summary>Test concurrent reconfig and stepdown</summary>
                <type id="3" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14718&amp;avatarType=issuetype">Task</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="jesse@mongodb.com">A. Jesse Jiryu Davis</assignee>
                                    <reporter username="jesse@mongodb.com">A. Jesse Jiryu Davis</reporter>
                        <labels>
                    </labels>
                <created>Mon, 24 Feb 2020 21:55:21 +0000</created>
                <updated>Sun, 29 Oct 2023 22:11:45 +0000</updated>
                            <resolved>Tue, 17 Mar 2020 13:24:14 +0000</resolved>
                                                    <fixVersion>4.7.0</fixVersion>
                                    <component>Replication</component>
                                        <votes>0</votes>
                                    <watches>3</watches>
                                                                                                                <comments>
                            <comment id="3093713" author="xgen-internal-githook" created="Mon, 18 May 2020 20:02:13 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Siyuan Zhou&apos;, &apos;email&apos;: &apos;visualzhou@gmail.com&apos;, &apos;username&apos;: &apos;visualzhou&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-48257&quot; title=&quot;Reject heartbeat reconfig when running for election&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-48257&quot;&gt;&lt;del&gt;SERVER-48257&lt;/del&gt;&lt;/a&gt; Backport handleHeartbeatResponse_forTest to 4.4&lt;/p&gt;

&lt;p&gt;The test helper is introduced in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-46381&quot; title=&quot;Test concurrent reconfig and stepdown&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-46381&quot;&gt;&lt;del&gt;SERVER-46381&lt;/del&gt;&lt;/a&gt; and updated by &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-48115&quot; title=&quot;Initialize pingStats before _handleHeartbeatResponse() in handleHeartbeatResponse_forTest()&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-48115&quot;&gt;&lt;del&gt;SERVER-48115&lt;/del&gt;&lt;/a&gt;.&lt;br/&gt;
Branch: v4.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/f5f77b5f38efa5c7254593d89e43ac23cd4fac0f&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/f5f77b5f38efa5c7254593d89e43ac23cd4fac0f&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2999754" author="siyuan.zhou@10gen.com" created="Wed, 25 Mar 2020 19:20:54 +0000"  >&lt;p&gt;We decided not to backport this test to 4.4, since it didn&apos;t uncover new bugs and the reconfig passthrough test suites in&#160;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-45094&quot; title=&quot;Add passthrough tests for safe reconfig&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-45094&quot;&gt;&lt;del&gt;SERVER-45094&lt;/del&gt;&lt;/a&gt; will give the same test coverage.&lt;/p&gt;</comment>
                            <comment id="2982920" author="xgen-internal-githook" created="Tue, 17 Mar 2020 12:49:54 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;A. Jesse Jiryu Davis&apos;, &apos;email&apos;: &apos;jesse@mongodb.com&apos;, &apos;username&apos;: &apos;ajdavis&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-46381&quot; title=&quot;Test concurrent reconfig and stepdown&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-46381&quot;&gt;&lt;del&gt;SERVER-46381&lt;/del&gt;&lt;/a&gt; Test concurrent stepdown and reconfig&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/8c1515929f34d41dbefbb9476e1dd893d523ad01&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/8c1515929f34d41dbefbb9476e1dd893d523ad01&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2980430" author="xgen-internal-githook" created="Sun, 15 Mar 2020 13:19:51 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;A. Jesse Jiryu Davis&apos;, &apos;username&apos;: &apos;ajdavis&apos;, &apos;email&apos;: &apos;jesse@mongodb.com&apos;}
&lt;p&gt;Message: Revert &quot;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-46381&quot; title=&quot;Test concurrent reconfig and stepdown&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-46381&quot;&gt;&lt;del&gt;SERVER-46381&lt;/del&gt;&lt;/a&gt; Test concurrent stepdown and reconfig&quot;&lt;/p&gt;

&lt;p&gt;This reverts commit 5b7782502396354468815ff56150be789599919a.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/ed023e8734948ebf89e87d7aa7ff4f1cbf4332fa&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/ed023e8734948ebf89e87d7aa7ff4f1cbf4332fa&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2979658" author="jesse" created="Sat, 14 Mar 2020 03:02:34 +0000"  >&lt;p&gt;Once &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-45096&quot; title=&quot;Test safe reconfig never rolls back entries committed in earlier configs.&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-45096&quot;&gt;&lt;del&gt;SERVER-45096&lt;/del&gt;&lt;/a&gt; is done, I can see if it&apos;s correct to remove a try/catch around this test&apos;s call to&#160;processReplSetReconfig.&lt;/p&gt;</comment>
                            <comment id="2979652" author="xgen-internal-githook" created="Sat, 14 Mar 2020 02:25:04 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;A. Jesse Jiryu Davis&apos;, &apos;username&apos;: &apos;ajdavis&apos;, &apos;email&apos;: &apos;jesse@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-46381&quot; title=&quot;Test concurrent reconfig and stepdown&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-46381&quot;&gt;&lt;del&gt;SERVER-46381&lt;/del&gt;&lt;/a&gt; Test concurrent stepdown and reconfig&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/5b7782502396354468815ff56150be789599919a&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/5b7782502396354468815ff56150be789599919a&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2968268" author="xgen-internal-githook" created="Wed, 11 Mar 2020 17:28:00 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;username&apos;: &apos;ldennis&apos;, &apos;name&apos;: &apos;Lingzhi Deng&apos;, &apos;email&apos;: &apos;lingzhi.deng@mongodb.com&apos;}
&lt;p&gt;Message: Revert &quot;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-46381&quot; title=&quot;Test concurrent reconfig and stepdown&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-46381&quot;&gt;&lt;del&gt;SERVER-46381&lt;/del&gt;&lt;/a&gt; Test concurrent reconfig and stepdown&quot;&lt;/p&gt;

&lt;p&gt;This reverts commit bdf61762f8fd755b784b55af8457f8fcdd7fe068.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/521ab081fdbd6a0ba5169f762a78e2158edba86c&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/521ab081fdbd6a0ba5169f762a78e2158edba86c&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2962624" author="xgen-internal-githook" created="Wed, 11 Mar 2020 01:11:52 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;username&apos;: &apos;ajdavis&apos;, &apos;name&apos;: &apos;A. Jesse Jiryu Davis&apos;, &apos;email&apos;: &apos;jesse@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-46381&quot; title=&quot;Test concurrent reconfig and stepdown&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-46381&quot;&gt;&lt;del&gt;SERVER-46381&lt;/del&gt;&lt;/a&gt; Test concurrent reconfig and stepdown&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/bdf61762f8fd755b784b55af8457f8fcdd7fe068&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/bdf61762f8fd755b784b55af8457f8fcdd7fe068&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2906502" author="jesse" created="Tue, 25 Feb 2020 03:36:14 +0000"  >&lt;p&gt;Scenario 4.&lt;/p&gt;

&lt;p&gt;Thread A is in&#160;ReplicationCoordinatorImpl::processReplSetReconfig, holding the replication coordinator mutex. It checks if self is primary (which it is), drops the mutex, stores the new config document, waits for the new config to propagate to a majority, and calls&#160;_performPostMemberStateUpdateAction, which I think must be&#160;kActionNone.&lt;/p&gt;

&lt;p&gt;Thread B is in ReplicationCoordinatorImpl::_handleHeartbeatResponse, holding the replication coordinator mutex. It sees a higher term in the response and calls&#160;TopologyCoordinator::prepareForUnconditionalStepDown(), schedules a call to&#160;_stepDownFinish, then drops the mutex.&lt;/p&gt;

&lt;p&gt;If the stepdown starts while the replSetReconfig command waits for a majority to replicate the new config, the command will return with an error. The reconfig may or may not eventually be committed. Some node in the set will run for election and win, and it may or may not have the new config. This non-determinism is by design.&lt;/p&gt;

&lt;p&gt;We can test the case where waiting for config commitment is interrupted by stepdown if we disconnect a quorum using mongobridge. That test can assert that some member is elected after the quorum is reconnected. It&apos;s acceptable for the primary to have the old or new config.&lt;/p&gt;</comment>
                            <comment id="2906280" author="jesse" created="Mon, 24 Feb 2020 22:58:20 +0000"  >&lt;p&gt;Scenario 3.&lt;/p&gt;

&lt;p&gt;Thread A is in ReplicationCoordinatorImpl::_handleHeartbeatResponse, holding the replication coordinator mutex. It sees a higher term in the response and calls&#160;TopologyCoordinator::prepareForUnconditionalStepDown(), schedules a call to&#160;_stepDownFinish, then drops the mutex.&lt;/p&gt;

&lt;p&gt;Thread B is in&#160;ReplicationCoordinatorImpl::processReplSetReconfig, holding the replication coordinator mutex. It checks if self is primary (which it is), drops the mutex, stores the new config document, and calls&#160;_performPostMemberStateUpdateAction (which cannot be stepdownSelf).&lt;/p&gt;

&lt;p&gt;A worker thread&#160;enters&#160;_stepDownFinish, takes the RSTL and replcoord mutex, and completes stepdown by calling&#160;_performPostMemberStateUpdateAction.&lt;/p&gt;

&lt;p&gt;We can test this sequence by enabling the&#160;blockHeartbeatStepdown failpoint until Thread B has finished.&lt;/p&gt;</comment>
                            <comment id="2906267" author="jesse" created="Mon, 24 Feb 2020 22:45:10 +0000"  >&lt;p&gt;Scenario 2 swaps the identities of Threads A and B, but it appears to have the same possible behaviors as Scenario 1.&lt;/p&gt;</comment>
                            <comment id="2906229" author="jesse" created="Mon, 24 Feb 2020 22:27:19 +0000"  >&lt;p&gt;Scenario 1.&lt;/p&gt;

&lt;p&gt;Thread A is in ReplicationCoordinatorImpl::_handleHeartbeatResponse, holding the replication coordinator mutex. It sees a higher term in the response and calls&#160;TopologyCoordinator::prepareForUnconditionalStepDown(), schedules a call to&#160;_stepDownFinish, then drops the mutex.&lt;/p&gt;

&lt;p&gt;Thread B is in ReplicationCoordinatorImpl::_handleHeartbeatResponse, holding the replication coordinator mutex. It&#160;sees a newer config and schedules a call to&#160;_heartbeatReconfigStore, then drops the mutex.&lt;/p&gt;

&lt;p&gt;Then the following two steps happen in some order:&lt;/p&gt;

&lt;p&gt;1. A worker thread enters&#160;_stepDownFinish, takes the RSTL and replcoord mutex, and completes stepdown by calling&#160;_performPostMemberStateUpdateAction.&lt;/p&gt;

&lt;p&gt;2. Another worker thread enters&#160;_heartbeatReconfigStore, which stores the local config, takes the mutex, and calls _heartbeatReconfigFinish. This cannot cause a stepdown (safe reconfig requires the primary remains electable). The thread drops the mutex and calls&#160;_performPostMemberStateUpdateAction.&lt;/p&gt;

&lt;p&gt;We can test ordering (1, 2) by introducing a failpoint blockHeartbeatReconfigStore at the top of _heartbeatReconfigStore. We can test ordering (2, 1) with the existing failpoint blockHeartbeatStepdown.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10520">
                    <name>Problem/Incident</name>
                                            <outwardlinks description="causes">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="1328258">SERVER-47758</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>12.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 11 Mar 2020 01:11:52 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        3 years, 38 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>PM-1209</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            3 years, 38 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_16465" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Linked BF Score</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>jesse@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>siyuan.zhou@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hwuitz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hwigw7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="3577">Repl 2020-03-09</customfieldvalue>
    <customfieldvalue id="3766">Repl 2020-03-23</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hwu53b:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>