<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:16:21 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-48178] Finding self in reconfig may be interrupted by closing connections due to rollback</title>
                <link>https://jira.mongodb.org/browse/SERVER-48178</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;When a node installs a new config, it needs to &lt;a href=&quot;https://github.com/mongodb/mongo/blob/eae31861e0f813f0099e1d490c4a622d75cd5a08/src/mongo/db/repl/isself.cpp#L221&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;create a new connection&lt;/a&gt; to find itself in the config. If the node start to roll back, the connection will be closed. Thus the node thinks it&apos;s not in this config and will transition to REMOVED. This issue becomes worse when authentication is enabled since more network roundtrips are involved.&lt;/p&gt;</description>
                <environment></environment>
        <key id="1349308">SERVER-48178</key>
            <summary>Finding self in reconfig may be interrupted by closing connections due to rollback</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="mark.benvenuto@mongodb.com">Mark Benvenuto</assignee>
                                    <reporter username="siyuan.zhou@mongodb.com">Siyuan Zhou</reporter>
                        <labels>
                            <label>safe-reconfig-related</label>
                    </labels>
                <created>Wed, 13 May 2020 07:23:04 +0000</created>
                <updated>Sun, 29 Oct 2023 22:08:16 +0000</updated>
                            <resolved>Wed, 8 Jul 2020 23:24:05 +0000</resolved>
                                                    <fixVersion>4.4.1</fixVersion>
                    <fixVersion>4.7.0</fixVersion>
                                                        <votes>0</votes>
                                    <watches>5</watches>
                                                                                                                <comments>
                            <comment id="3320051" author="xgen-internal-githook" created="Tue, 4 Aug 2020 16:44:27 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Mark Benvenuto&apos;, &apos;email&apos;: &apos;mark.benvenuto@mongodb.com&apos;, &apos;username&apos;: &apos;markbenvenuto&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-48178&quot; title=&quot;Finding self in reconfig may be interrupted by closing connections due to rollback&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-48178&quot;&gt;&lt;del&gt;SERVER-48178&lt;/del&gt;&lt;/a&gt; Finding self in reconfig may be interrupted by closing connections due to rollback&lt;/p&gt;

&lt;p&gt;(cherry picked from commit acd6c3d067a90f6cf37b6802160b505f7935218b)&lt;br/&gt;
Branch: v4.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/48bcef9c841af51a146f789c0ec92cf7fa6b4a6e&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/48bcef9c841af51a146f789c0ec92cf7fa6b4a6e&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3283011" author="pavithra.vetriselvan" created="Mon, 13 Jul 2020 18:06:07 +0000"  >&lt;p&gt;Nope, it&apos;s not a blocker, just a bug fix! 4.4.1 should be fine. &lt;/p&gt;</comment>
                            <comment id="3282971" author="mark.benvenuto" created="Mon, 13 Jul 2020 17:47:48 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=pavithra.vetriselvan&quot; class=&quot;user-hover&quot; rel=&quot;pavithra.vetriselvan&quot;&gt;pavithra.vetriselvan&lt;/a&gt;, do we need the fix for 4.4.0?&lt;/p&gt;</comment>
                            <comment id="3282951" author="pavithra.vetriselvan" created="Mon, 13 Jul 2020 17:41:15 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=mark.benvenuto&quot; class=&quot;user-hover&quot; rel=&quot;mark.benvenuto&quot;&gt;mark.benvenuto&lt;/a&gt; This is happening on the 4.4 branch as well, so could we request a backport? I&apos;ve linked the relevant BF.&lt;/p&gt;</comment>
                            <comment id="3276087" author="xgen-internal-githook" created="Wed, 8 Jul 2020 20:50:50 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Mark Benvenuto&apos;, &apos;email&apos;: &apos;mark.benvenuto@mongodb.com&apos;, &apos;username&apos;: &apos;markbenvenuto&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-48178&quot; title=&quot;Finding self in reconfig may be interrupted by closing connections due to rollback&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-48178&quot;&gt;&lt;del&gt;SERVER-48178&lt;/del&gt;&lt;/a&gt; Finding self in reconfig may be interrupted by closing connections due to rollback&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/acd6c3d067a90f6cf37b6802160b505f7935218b&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/acd6c3d067a90f6cf37b6802160b505f7935218b&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3155673" author="siyuan.zhou@10gen.com" created="Mon, 1 Jun 2020 20:51:18 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=judah.schvimer&quot; class=&quot;user-hover&quot; rel=&quot;judah.schvimer&quot;&gt;judah.schvimer&lt;/a&gt;, good point. Updated.&lt;/p&gt;</comment>
                            <comment id="3151643" author="judah.schvimer" created="Mon, 1 Jun 2020 18:30:44 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=siyuan.zhou&quot; class=&quot;user-hover&quot; rel=&quot;siyuan.zhou&quot;&gt;siyuan.zhou&lt;/a&gt;, should this no longer be a replication quick win?&lt;/p&gt;</comment>
                            <comment id="3107703" author="siyuan.zhou@10gen.com" created="Tue, 26 May 2020 19:08:03 +0000"  >&lt;p&gt;I looked more into the comment &quot;We need to avoid the isMaster call triggered by a normal connect&quot;. IIUC, it&apos;s out of date. I didn&apos;t find any caller of isSelf function within the replication mutex. If it&apos;s called within the repl mutex, authentication should have triggered deadlock reliably. We have sufficient test coverage for that, so I&apos;m not concerned.&lt;/p&gt;</comment>
                            <comment id="3107679" author="siyuan.zhou@10gen.com" created="Tue, 26 May 2020 18:59:54 +0000"  >&lt;p&gt;Thanks for investigating into this! I feel I don&apos;t have the context of whether hardcoding&#160;SCRAM-SHA-256 is fine, but I trust Security team on that. Assigning to Security team.&lt;/p&gt;</comment>
                            <comment id="3107530" author="spencer.jackson@10gen.com" created="Tue, 26 May 2020 18:11:43 +0000"  >&lt;p&gt;Looking at this some more, I think that the use cases which require performing intracluster mechanism negotiation use the connection_pool_tl. That suggests that it should be safe to remove the isMaster call from client/authenticate.cpp, and hardcode SCRAM-SHA-256. That would leave the connection tags alone, as the replication system expects.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=siyuan.zhou&quot; class=&quot;user-hover&quot; rel=&quot;siyuan.zhou&quot;&gt;siyuan.zhou&lt;/a&gt;, would you want to assign this over to security to make that change?&lt;/p&gt;</comment>
                            <comment id="3102455" author="siyuan.zhou@10gen.com" created="Thu, 21 May 2020 16:01:44 +0000"  >&lt;p&gt;Right, it should be &lt;tt&gt;hangUpOnStepDown: false&lt;/tt&gt;. Sorry I missed that. All internal connections that go through&#160;TLConnection will attach the flag.&lt;/p&gt;
&lt;blockquote&gt;&lt;p&gt;Do you have any reason to believe that under some circumstances, that flag should&#160;&lt;b&gt;not&lt;/b&gt;&#160;be set?&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;I think internal connections of replication system do want to avoid closing connection. I&apos;m not sure of sharding connections. The flag is not set on connections from drivers or the shell - those external connections. We close connections in fewer cases on stepdown now since we prefer to keep the reads or kill the writes. However, rollback or removing the node still close all connections that&apos;s not kKeepOpen as you mentioned since a rollback node cannot even serve reads. The flag name is misleading.&lt;/p&gt;</comment>
                            <comment id="3102230" author="spencer.jackson@10gen.com" created="Thu, 21 May 2020 14:29:25 +0000"  >&lt;p&gt;Looking into this some more, I think there might be a couple of scenarios where we still do need to rely on negotiation during intracluster authentication... so we probably still do need to call isMaster in the authentication path.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=siyuan.zhou&quot; class=&quot;user-hover&quot; rel=&quot;siyuan.zhou&quot;&gt;siyuan.zhou&lt;/a&gt;, I realize that I may have said the wrong thing earlier, would the isMaster invocation need to pass &lt;tt&gt;hangUpOnStepDown: true&lt;/tt&gt; or &lt;tt&gt;hangUpOnStepDown: false&lt;/tt&gt;?&lt;/p&gt;

&lt;p&gt;Do you have any reason to believe that under some circumstances, that flag should &lt;b&gt;not&lt;/b&gt; be set?&lt;/p&gt;</comment>
                            <comment id="3098528" author="siyuan.zhou@10gen.com" created="Wed, 20 May 2020 22:54:24 +0000"  >&lt;blockquote&gt;&lt;p&gt;Or does it kill everything not in the kKeepOpen state?&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;Oh, yes! Thank you.&lt;/p&gt;

&lt;p&gt;I assume isMaster is the first step in authentication, so we could pass &lt;tt&gt;&apos;hangUpOnStepDown: true&apos;&lt;/tt&gt;. Since we always &lt;a href=&quot;https://github.com/mongodb/mongo/blob/b77252a641838f43e3c09796a09be22ab8f44e1c/src/mongo/transport/service_state_machine.cpp#L638&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;skip closing connection in kPending state&lt;/a&gt;, I wonder why &lt;a href=&quot;https://github.com/mongodb/mongo/blob/eae81ac42abf876039042d3f0e08e319e7e2da49/src/mongo/client/authenticate.cpp#L259&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;the isMaster invocation&lt;/a&gt; you linked above doesn&apos;t &lt;a href=&quot;https://github.com/mongodb/mongo/blob/eae31861e0f813f0099e1d490c4a622d75cd5a08/src/mongo/executor/connection_pool_tl.cpp#L156&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;attach the flag&lt;/a&gt; by default.&lt;/p&gt;

&lt;p&gt;As a separate note, my understanding is we wanted to avoid a deadlock in isSelf, so we deliberately skip isMaster in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-19035&quot; title=&quot;Implement OP_COMMAND autodetection in the clientdriver&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-19035&quot;&gt;&lt;del&gt;SERVER-19035&lt;/del&gt;&lt;/a&gt;. Since it doesn&apos;t call isMaster, the node is in kPending state and avoided closing connections.&lt;/p&gt;
&lt;blockquote&gt;&lt;p&gt;// We need to avoid the isMaster call triggered by a normal connect, which would&lt;br/&gt;
 // cause a deadlock. &apos;isSelf&apos; is called by the Replication Coordinator when validating&lt;br/&gt;
 // a replica set configuration document, but the &apos;isMaster&apos; command requires a lock on the&lt;br/&gt;
 // replication coordinator to execute. As such we call we call &apos;connectSocketOnly&apos;, which&lt;br/&gt;
 // does not call &apos;isMaster&apos;.&lt;/p&gt;&lt;/blockquote&gt;</comment>
                            <comment id="3098401" author="spencer.jackson@10gen.com" created="Wed, 20 May 2020 21:34:37 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=siyuan.zhou&quot; class=&quot;user-hover&quot; rel=&quot;siyuan.zhou&quot;&gt;siyuan.zhou&lt;/a&gt;, are you sure that rollBack kills everything in the kKeepOpen state? Or does it &lt;a href=&quot;https://github.com/mongodb/mongo/blob/25c694f365db0f07a445bd17b6cd5cbf32f5f2f9/src/mongo/transport/service_entry_point_impl.cpp#L208-L209&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;kill everything&lt;/a&gt; &lt;b&gt;not&lt;/b&gt; in the kKeepOpen state? &lt;/p&gt;

&lt;p&gt;I believe that this command path can &lt;a href=&quot;https://github.com/mongodb/mongo/blob/eae81ac42abf876039042d3f0e08e319e7e2da49/src/mongo/client/authenticate.cpp#L259&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;invoke isMaster&lt;/a&gt; during authentication, in order to perform SCRAM mechanism negotiation. However, now that all nodes in the current and previous release support SCRAM-SHA-256 for intracluster auth, we could hardcode the mechanism and avoid the isMaster roundtrip.&lt;/p&gt;

&lt;p&gt;However, that might not be everything that you need, if &apos;&lt;tt&gt;hangUpOnStepDown: true&lt;/tt&gt;&apos; has be set on an isMaster invokation. Maybe authenticateInternalUser should stop calling isMaster, and you should start calling it in isSelf?&lt;/p&gt;</comment>
                            <comment id="3093476" author="siyuan.zhou@10gen.com" created="Mon, 18 May 2020 18:27:43 +0000"  >&lt;p&gt;I worked around this problem by not making concurrent reconfig and rollback more likely in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-48102&quot; title=&quot;Update heartbeat state on primary even if heartbeat response includes a new config&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-48102&quot;&gt;&lt;del&gt;SERVER-48102&lt;/del&gt;&lt;/a&gt;. This issue still exists elsewhere.&lt;/p&gt;

&lt;p&gt;One solution is to avoid closing connection during authentication and isSelf command.&lt;/p&gt;

&lt;p&gt;Rollback &lt;a href=&quot;https://github.com/mongodb/mongo/blob/912735588386424712a1525da1574a4554bf1787/src/mongo/db/repl/replication_coordinator_impl.cpp#L4113&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;closes&lt;/a&gt; connections &lt;a href=&quot;https://github.com/mongodb/mongo/blob/25c694f365db0f07a445bd17b6cd5cbf32f5f2f9/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp#L748&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;in kKeepOpen state&lt;/a&gt;. kKeepOpen state is only &lt;a href=&quot;https://github.com/mongodb/mongo/blob/845b52ae1c13f10d79993950888819347fac3aa3/src/mongo/db/repl/replication_info.cpp#L339&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;set on isMaster&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=spencer.jackson&quot; class=&quot;user-hover&quot; rel=&quot;spencer.jackson&quot;&gt;spencer.jackson&lt;/a&gt;, I&apos;m not sure why the connection for isSelf can still be closed. I don&apos;t think we run isMaster for &lt;a href=&quot;https://github.com/mongodb/mongo/blob/eae31861e0f813f0099e1d490c4a622d75cd5a08/src/mongo/db/repl/isself.cpp#L224-L231&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;isSelf check&lt;/a&gt;&#160;at all. This issue only showed up when auth is enabled, which I thought is due to a wider race window.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="1347174">SERVER-48102</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1349322">SERVER-48179</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="560807">SERVER-35649</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1404709">SERVER-49388</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1400628">SERVER-49305</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>15.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="18953"><![CDATA[v4.4]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 20 May 2020 21:34:37 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        3 years, 27 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            3 years, 27 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_16465" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Linked BF Score</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>14.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>judah.schvimer@mongodb.com</customfieldvalue>
            <customfieldvalue>mark.benvenuto@mongodb.com</customfieldvalue>
            <customfieldvalue>pavithra.vetriselvan@mongodb.com</customfieldvalue>
            <customfieldvalue>siyuan.zhou@mongodb.com</customfieldvalue>
            <customfieldvalue>spencer.jackson@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hxkzmn:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hxc59z:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="4035">Security 2020-06-29</customfieldvalue>
    <customfieldvalue id="4036">Security 2020-07-13</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hxklvz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>