<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:40:14 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-35567] Concurrency simultaneous replication tests - unblacklist snapshot_read_kill_operations.js and remove snapshot_read_kill_op_only.js</title>
                <link>https://jira.mongodb.org/browse/SERVER-35567</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;The test &lt;tt&gt;snapshot_read_kill_operations.js&lt;/tt&gt; is blacklisted in the &lt;tt&gt;concurrency_simultaneous_replication&lt;/tt&gt; suite, due to states that are not supported in 4.0. An extended test was created, &lt;tt&gt;snapshot_read_kill_op_only.js&lt;/tt&gt;.&lt;/p&gt;

&lt;p&gt;The extended test and blacklist can be removed once the global cursor management has been implemented.&lt;/p&gt;</description>
                <environment></environment>
        <key id="558227">SERVER-35567</key>
            <summary>Concurrency simultaneous replication tests - unblacklist snapshot_read_kill_operations.js and remove snapshot_read_kill_op_only.js</summary>
                <type id="3" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14718&amp;avatarType=issuetype">Task</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="2">Won&apos;t Fix</resolution>
                                        <assignee username="anton.korshunov@mongodb.com">Anton Korshunov</assignee>
                                    <reporter username="jonathan.abrahams">Jonathan Abrahams</reporter>
                        <labels>
                            <label>open_todo_in_code</label>
                            <label>todo_in_code</label>
                    </labels>
                <created>Tue, 12 Jun 2018 20:55:48 +0000</created>
                <updated>Wed, 6 Nov 2019 12:28:11 +0000</updated>
                            <resolved>Mon, 4 Mar 2019 11:30:20 +0000</resolved>
                                                                    <component>Testing Infrastructure</component>
                                        <votes>0</votes>
                                    <watches>6</watches>
                                                                                                                <comments>
                            <comment id="2519585" author="xgen-internal-githook" created="Wed, 6 Nov 2019 12:28:11 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;username&apos;: &apos;antkorsh&apos;, &apos;email&apos;: &apos;anton.korshunov@mongodb.com&apos;, &apos;name&apos;: &apos;Anton Korshunov&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-44212&quot; title=&quot;Complete TODO listed in SERVER-35567&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-44212&quot;&gt;&lt;del&gt;SERVER-44212&lt;/del&gt;&lt;/a&gt; Complete TODO listed in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-35567&quot; title=&quot;Concurrency simultaneous replication tests - unblacklist snapshot_read_kill_operations.js and remove snapshot_read_kill_op_only.js&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-35567&quot;&gt;&lt;del&gt;SERVER-35567&lt;/del&gt;&lt;/a&gt;&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/246e119f7258784535b2eee9fc8645fd382a2a51&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/246e119f7258784535b2eee9fc8645fd382a2a51&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2169065" author="anton.korshunov" created="Mon, 4 Mar 2019 11:29:55 +0000"  >&lt;p&gt;So, after all, it was a combination of a test issue along with a design of the killCursors locking protocol. The hang that I was observing was caused by the implementation of the FSM killSessions state. At the beginning of the test we start a new transaction and insert a session ID into a collection. The reason we need to store session IDs in a collection is because all FSM threads need access this information in order to randomly pick a session ID to be killed. However, the FSM framework doesn&apos;t provide a global working area which could be accessed by all threads, so&#160;FSM workloads can only operate local structures allocated per thread. In the killSessions state we execute a find command (which takes locks) to select the session ID from the collection, and this is where we get stuck due to the following sequence of lock acquisitions:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;Thread 1 starts a transactions and takes Global IX lock to write down the session ID&lt;/li&gt;
	&lt;li&gt;Thread 2 in a parallel test executes a DDL command and queues Global X lock&lt;/li&gt;
	&lt;li&gt;Thread 1 enters killSessions state, runs find and queues Global IS lock&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;To workaround this issue I modified the test to store the sessions IDs in a separate mongod instance. Then I hit another problem with the test. Here is how we pick a session to kill:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: #006699; font-weight: bold; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;while&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; (!sessionDocToKill || idToKill == &lt;/span&gt;&lt;span style=&quot;color: blue; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&quot;sessionDoc&quot;&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; + &lt;/span&gt;&lt;span style=&quot;color: #006699; font-weight: bold; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;this&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;.tid) {&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&#160; &#160;idToKill = &lt;/span&gt;&lt;span style=&quot;color: blue; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&quot;sessionDoc&quot;&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; + Math.floor(Math.random() * &lt;/span&gt;&lt;span style=&quot;color: #006699; font-weight: bold; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;this&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;.threadCount);&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&#160; &#160;sessionDocToKill = db[collName].findOne({&lt;/span&gt;&lt;span style=&quot;color: blue; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&quot;_id&quot;&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;: idToKill});&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;A sessions ID in this test is formed as &lt;em&gt;&quot;sessionId&quot; + tid&lt;/em&gt;, where &lt;em&gt;tid&lt;/em&gt; is a thread ID passed to the test by the FSM workload manager. If we were running 5 threads with threads IDs between &lt;span class=&quot;error&quot;&gt;&amp;#91;60...64&amp;#93;&lt;/span&gt;, we&apos;d never be able to exit the loop above, as the &lt;em&gt;Math.floor(...)&lt;/em&gt; expression would return a value in the range &lt;span class=&quot;error&quot;&gt;&amp;#91;0..4&amp;#93;&lt;/span&gt;. I don&apos;t know why this part was written the way it was, as a less error-prone approach would be to find all documents in the collections holding the sessionID values, and then randomly select one element in the returned array. This is how I re-wrote it and it worked fine.&lt;/p&gt;

&lt;p&gt;Then I hit another deadlock, and this time it was caused by the codepath I mentioned earlier. When the &lt;em&gt;killSessions&lt;/em&gt; command tried to kill a cursor we acquire a Global IS lock in&#160;AutoStatsTracker, and we hang by the same reason I described earlier.&lt;/p&gt;

&lt;p&gt;For that reason, the test cannot be re-enabled. Also, after talking to &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=david.storch&quot; class=&quot;user-hover&quot; rel=&quot;david.storch&quot;&gt;david.storch&lt;/a&gt; we decided that supporting this particular scenario is not worth the additional server changes to avoid taking the lock there. At least, not at this time. Instead, we&apos;ll create a separate ticket to address this issue in the future: see &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-39939&quot; title=&quot;Avoid taking any lock manager locks in the killCursors code path&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-39939&quot;&gt;SERVER-39939&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;</comment>
                            <comment id="2166650" author="craig.homa" created="Thu, 28 Feb 2019 20:09:30 +0000"  >&lt;p&gt;Removed this from the All Cursors Globally Managed epic as further testing showed that the work done in the project did not enable the extra testing.&lt;/p&gt;</comment>
                            <comment id="2163208" author="tess.avitabile" created="Tue, 26 Feb 2019 17:08:13 +0000"  >&lt;p&gt;I was referring to the find in the &lt;tt&gt;killSessions&lt;/tt&gt; FSM state. Can you see the JS stack traces for the FSM states?&lt;/p&gt;</comment>
                            <comment id="2163201" author="anton.korshunov" created="Tue, 26 Feb 2019 17:05:52 +0000"  >&lt;p&gt;No, I don&apos;t see the killSessions command in the&#160;hang analyzer output, which means it completed successfully. I&apos;ll send you the hang analyzer output if you want to take a look at it.&lt;/p&gt;</comment>
                            <comment id="2163157" author="tess.avitabile" created="Tue, 26 Feb 2019 16:48:57 +0000"  >&lt;p&gt;Without looking at the hang analyzer output myself, it&apos;s hard to say for sure what is going on. However, I do see one thing in the &lt;tt&gt;killSessions&lt;/tt&gt; FSM state that is suspicious. We select the session to kill by doing a &lt;a href=&quot;https://github.com/mongodb/mongo/blob/b0d3c0d2934a2096ba27362db22fe768b5341485/jstests/concurrency/fsm_workloads/snapshot_read_kill_operations.js#L61&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;find&lt;/a&gt;, which takes locks. If there is a DDL operation that is blocked by an open transaction, then this could cause the find to hang. Do you see that the &lt;tt&gt;killSessions&lt;/tt&gt; state is getting stuck at this point?&lt;/p&gt;

&lt;p&gt;If that doesn&apos;t help diagnose the problem, I&apos;d be happy to look at the hang analyzer output with you.&lt;/p&gt;</comment>
                            <comment id="2162988" author="anton.korshunov" created="Tue, 26 Feb 2019 15:17:40 +0000"  >&lt;p&gt;A further update. We had a theory that the deadlock could still be related to&#160;killCursors command acquiring a lock in AutoStatsTracker, but I couldn&apos;t find an evidence that is was the cause of the deadlock. I actually found out the opposite, that neither killOp nor killCursors are the cause of the hang. The suite only fails when killSessions comes into play. So, if the issue was with the cursor manager and collection locks, then it would also manifest in killCursors, which is not the case. That said, it could be something in killSessions which leads to a deadlock (e.g., a transaction is aborted without a proper clean up). I wonder if someone from the Replication team could comment on this matter.&lt;/p&gt;</comment>
                            <comment id="1939597" author="tess.avitabile" created="Thu, 5 Jul 2018 16:31:23 +0000"  >&lt;p&gt;The test &lt;tt&gt;snapshot_read_kill_operations.js&lt;/tt&gt; alternates running transaction operations, &lt;tt&gt;killOp&lt;/tt&gt;, &lt;tt&gt;killCursors&lt;/tt&gt;, and &lt;tt&gt;killSessions&lt;/tt&gt;. If there are concurrent DDL operations happening in the suite that are blocked by transactions, then the test can hang in the &lt;tt&gt;killCursors&lt;/tt&gt; or &lt;tt&gt;killSessions&lt;/tt&gt; state, since killing cursors requires taking collection locks. Since the test hangs in &lt;tt&gt;killCursors&lt;/tt&gt; or &lt;tt&gt;killSessions&lt;/tt&gt;, it will not progress to a state where it would abort/commit the transaction, which would allow the DDL operation to proceed.&#160;In Evergreen, we increase the transaction expiration deadline to 2 hours, so these hangs will not be resolved by the transaction reaper. When we no longer require taking collection locks to kill cursors, it should be possible to re-enable this test in the &lt;tt&gt;concurrency_simultaneous_replication&lt;/tt&gt; suite, since it should no longer hange in &lt;tt&gt;killCursors&lt;/tt&gt; or &lt;tt&gt;killSessions&lt;/tt&gt;.&lt;/p&gt;</comment>
                            <comment id="1939542" author="jonathan.abrahams" created="Thu, 5 Jul 2018 16:08:00 +0000"  >&lt;p&gt;I believe &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=tess.avitabile&quot; class=&quot;user-hover&quot; rel=&quot;tess.avitabile&quot;&gt;tess.avitabile&lt;/a&gt; should have more context on that. I believe that random &lt;tt&gt;killSessions&lt;/tt&gt; and &lt;tt&gt;killCursors&lt;/tt&gt; created deadlocks when subsequently committing or aborting the associated transaction.&lt;/p&gt;</comment>
                            <comment id="1939525" author="david.storch" created="Thu, 5 Jul 2018 15:59:35 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=jonathan.abrahams&quot; class=&quot;user-hover&quot; rel=&quot;jonathan.abrahams&quot;&gt;jonathan.abrahams&lt;/a&gt;, can you elaborate on why this test is blacklisted from concurrency_simultaneous_replication? That may help me understand why the query team&apos;s planned work around cursor management will allow us to unblacklist the test.&lt;/p&gt;</comment>
                            <comment id="1939503" author="jonathan.abrahams" created="Thu, 5 Jul 2018 15:49:42 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=david.storch&quot; class=&quot;user-hover&quot; rel=&quot;david.storch&quot;&gt;david.storch&lt;/a&gt; Yeah that is a bit confusing! When the work to support the globally managed cursors is implemented, then this workload does not need to blacklisted anymore.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=tess.avitabile&quot; class=&quot;user-hover&quot; rel=&quot;tess.avitabile&quot;&gt;tess.avitabile&lt;/a&gt; Is there a particular SERVER ticket that this should be dependent on?&lt;/p&gt;</comment>
                            <comment id="1938417" author="david.storch" created="Tue, 3 Jul 2018 21:30:52 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=jonathan.abrahams&quot; class=&quot;user-hover&quot; rel=&quot;jonathan.abrahams&quot;&gt;jonathan.abrahams&lt;/a&gt;, this ticket is in an odd state. Its component is &quot;Testing Infrastructure&quot;, yet it is assigned to the replication team, but it is also inside an Epic for a query team project. Can you clarify? Which team do you imagine will do this work?&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="935589">SERVER-43479</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="978507">SERVER-44212</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="708629">SERVER-39939</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>12.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 3 Jul 2018 21:30:52 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        4 years, 14 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>xgen-internal-githook</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            4 years, 14 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>anton.korshunov@mongodb.com</customfieldvalue>
            <customfieldvalue>craig.homa@mongodb.com</customfieldvalue>
            <customfieldvalue>david.storch@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>jonathan.abrahams</customfieldvalue>
            <customfieldvalue>tess.avitabile@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hu0f1j:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr8blz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="2788">Query 2019-02-25</customfieldvalue>
    <customfieldvalue id="2789">Query 2019-03-11</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hu01av:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>