<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:26:48 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-31368] Log time spent waiting for other shards in merge cursors aggregation stage</title>
                <link>https://jira.mongodb.org/browse/SERVER-31368</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Aggregation queries using $mergeCursors communicate with other shards and therefore can be affected by communication issues or issues on other nodes. In the following example the query ran for a very long time but did not spend any time acquiring locks and did not yield. Possibly it was waiting for a reponse from one or more of the other hosts, but there is no indication of that in the log message. It would be helpful for diagnosis if the slow query report indicated how much time was spent waiting for each host.&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-06-23T06:53:15.357+0200 I COMMAND  [conn44] command ... command: aggregate { aggregate: &quot;...&quot;, pipeline: [ { $mergeCursors: [ { host: &quot;...&quot;, ns: &quot;...&quot;, id: 253733414549 }, ...]}, { $group: ... } keysExamined:0 docsExamined:0 numYields:0 nreturned:0 reslen:17527 locks:{ Global: { acquireCount: { r: 6 } }, Database: { acquireCount: { r: 3 } }, Collection: { acquireCount: { r: 2 } } } protocol:op_command 2909221ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</description>
                <environment></environment>
        <key id="439486">SERVER-31368</key>
            <summary>Log time spent waiting for other shards in merge cursors aggregation stage</summary>
                <type id="4" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14710&amp;avatarType=issuetype">Improvement</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="david.percy@mongodb.com">David Percy</assignee>
                                    <reporter username="bruce.lucas@mongodb.com">Bruce Lucas</reporter>
                        <labels>
                            <label>SWDI</label>
                            <label>qopt-team</label>
                            <label>storch</label>
                    </labels>
                <created>Tue, 3 Oct 2017 18:19:27 +0000</created>
                <updated>Mon, 30 Oct 2023 23:13:09 +0000</updated>
                            <resolved>Thu, 23 Apr 2020 21:50:56 +0000</resolved>
                                                    <fixVersion>4.4.0-rc5</fixVersion>
                    <fixVersion>4.7.0</fixVersion>
                    <fixVersion>4.2.10</fixVersion>
                                    <component>Aggregation Framework</component>
                    <component>Diagnostics</component>
                                        <votes>30</votes>
                                    <watches>30</watches>
                                                                                                                <comments>
                            <comment id="3392813" author="xgen-internal-githook" created="Mon, 14 Sep 2020 17:53:01 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;David Percy&apos;, &apos;email&apos;: &apos;david.percy@mongodb.com&apos;, &apos;username&apos;: &apos;dpercy&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-31368&quot; title=&quot;Log time spent waiting for other shards in merge cursors aggregation stage&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-31368&quot;&gt;&lt;del&gt;SERVER-31368&lt;/del&gt;&lt;/a&gt; Tag log_remote_op_wait.js&lt;br/&gt;
Branch: v4.2&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/220d72da13180652f4986bc65a0dd95966973dd0&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/220d72da13180652f4986bc65a0dd95966973dd0&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3384376" author="xgen-internal-githook" created="Wed, 9 Sep 2020 23:27:23 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;David Percy&apos;, &apos;email&apos;: &apos;david.percy@mongodb.com&apos;, &apos;username&apos;: &apos;dpercy&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-31368&quot; title=&quot;Log time spent waiting for other shards in merge cursors aggregation stage&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-31368&quot;&gt;&lt;del&gt;SERVER-31368&lt;/del&gt;&lt;/a&gt; Make regex more permissive in log_remote_op_wait.js&lt;/p&gt;

&lt;p&gt;This was only failing on Windows, so I suspect there was a \r character at&lt;br/&gt;
the end of the line. The important property to test is that the line&apos;s last&lt;br/&gt;
field says &quot;42ms&quot; or similar, so allowing zero or more whitespace at the end&lt;br/&gt;
seems fine.&lt;br/&gt;
Branch: v4.2&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/33887c37750c267f7ddfdeada7096d6abc0bf128&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/33887c37750c267f7ddfdeada7096d6abc0bf128&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3384375" author="xgen-internal-githook" created="Wed, 9 Sep 2020 23:27:17 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;David Percy&apos;, &apos;email&apos;: &apos;david.percy@mongodb.com&apos;, &apos;username&apos;: &apos;dpercy&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-31368&quot; title=&quot;Log time spent waiting for other shards in merge cursors aggregation stage&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-31368&quot;&gt;&lt;del&gt;SERVER-31368&lt;/del&gt;&lt;/a&gt; Exclude log_remote_op_wait.js from last_stable&lt;br/&gt;
Branch: v4.2&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/46d96188b11b792f976eabe35b0f8b16d5a6f202&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/46d96188b11b792f976eabe35b0f8b16d5a6f202&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3380730" author="xgen-internal-githook" created="Tue, 8 Sep 2020 19:53:00 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;David Percy&apos;, &apos;email&apos;: &apos;david.percy@mongodb.com&apos;, &apos;username&apos;: &apos;dpercy&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-31368&quot; title=&quot;Log time spent waiting for other shards in merge cursors aggregation stage&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-31368&quot;&gt;&lt;del&gt;SERVER-31368&lt;/del&gt;&lt;/a&gt; Log time spent waiting in BlockingResultsMerger&lt;br/&gt;
Branch: v4.2&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/cd92e6054c21f744397d3c2f3fc8c19d91564199&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/cd92e6054c21f744397d3c2f3fc8c19d91564199&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3074768" author="xgen-internal-githook" created="Thu, 7 May 2020 21:05:43 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;David Percy&apos;, &apos;email&apos;: &apos;david.percy@mongodb.com&apos;, &apos;username&apos;: &apos;dpercy&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-31368&quot; title=&quot;Log time spent waiting for other shards in merge cursors aggregation stage&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-31368&quot;&gt;&lt;del&gt;SERVER-31368&lt;/del&gt;&lt;/a&gt; Log time spent waiting in BlockingResultsMerger&lt;br/&gt;
Branch: v4.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/ed15e4756564769d4ce98ad8910a878ea5cef639&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/ed15e4756564769d4ce98ad8910a878ea5cef639&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3053474" author="xgen-internal-githook" created="Thu, 23 Apr 2020 20:36:55 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;David Percy&apos;, &apos;email&apos;: &apos;david.percy@mongodb.com&apos;, &apos;username&apos;: &apos;dpercy&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-31368&quot; title=&quot;Log time spent waiting for other shards in merge cursors aggregation stage&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-31368&quot;&gt;&lt;del&gt;SERVER-31368&lt;/del&gt;&lt;/a&gt; Log time spent waiting in BlockingResultsMerger&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/d7c8f19c0980713d38a975d5b546f9b051e19649&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/d7c8f19c0980713d38a975d5b546f9b051e19649&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2235053" author="bruce.lucas@10gen.com" created="Mon, 6 May 2019 11:40:25 +0000"  >&lt;blockquote&gt;
&lt;p&gt;&quot;mergeCursorWaitMS&quot;&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;With a couple exceptions the most used convention in mongod seems to be &quot;...Micros&quot; and &quot;...Millis&quot;; should we follow that here?&lt;/p&gt;</comment>
                            <comment id="2133205" author="bruce.lucas@10gen.com" created="Thu, 31 Jan 2019 13:26:38 +0000"  >&lt;p&gt;Yes, it should be logged at level 0 so we can do RCAs on events that may not be possible to reproduce.&lt;/p&gt;</comment>
                            <comment id="2132799" author="bernard.gorman" created="Wed, 30 Jan 2019 23:48:48 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=david.storch&quot; class=&quot;user-hover&quot; rel=&quot;david.storch&quot;&gt;david.storch&lt;/a&gt; we&apos;ll have to do the work to record this metric in either case, so I don&apos;t see any disadvantage to making it part of the regular slow query line. Putting it at LOG(1) also means that TSEs would almost certainly need to ask users to increase their &lt;tt&gt;logLevel&lt;/tt&gt; and re-run the aggregation in order to generate the line, which may not be feasible if the aggregation in question was extremely long-running or disruptive to their system.&lt;/p&gt;</comment>
                            <comment id="2132793" author="david.storch" created="Wed, 30 Jan 2019 23:43:11 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bernard.gorman&quot; class=&quot;user-hover&quot; rel=&quot;bernard.gorman&quot;&gt;bernard.gorman&lt;/a&gt; will this be part of the regular slow query log line, or will we introduce a separate log line at logLevel 1 with this information?&lt;/p&gt;</comment>
                            <comment id="1691533" author="david.storch" created="Fri, 6 Oct 2017 15:29:33 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt;, understood. I imagine that when we implement slow query logging on mongos we will have to solve the problem of how to log information about the latencies associated with each shard. The information should be presented in the same way irrespective of whether the optimization logic decides to merge the result set on mongos or on one of the shards. So &quot;Log slow queries on mongos&quot; is really a bit of a misnomer. A more appropriate name would be &quot;Log slow queries in the sharded query execution path&quot;.&lt;/p&gt;</comment>
                            <comment id="1691445" author="bruce.lucas@10gen.com" created="Fri, 6 Oct 2017 14:40:03 +0000"  >&lt;p&gt;I noticed this was made part of &quot;Log slow queries on mongos&quot;, but just to be clear, this particular issue is about what we log on mongod, not on mongos.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10320">
                    <name>Documented</name>
                                                                <inwardlinks description="is documented by">
                                        <issuelink>
            <issuekey id="1327457">DOCS-13613</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10520">
                    <name>Problem/Incident</name>
                                            <outwardlinks description="causes">
                                        <issuelink>
            <issuekey id="1589123">SERVER-53751</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="1342782">SERVER-48001</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1194559">SERVER-46446</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="398185">SERVER-29854</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>12.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>14.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="18953"><![CDATA[v4.4]]></customfieldvalue>
    <customfieldvalue key="16775"><![CDATA[v4.2]]></customfieldvalue>
    <customfieldvalue key="15640"><![CDATA[v4.0]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[5002K00000kpfsuQAA]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 6 Oct 2017 15:29:33 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        3 years, 21 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_17052" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Downstream Changes Summary</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>This change adds a new optional field, &amp;quot;remoteOpWaitMillis&amp;quot; to the profiler / slow query log lines.  It tells you how much time the node spent waiting for results from other nodes.  By comparing this with the total &amp;quot;durationMillis&amp;quot; you can figure out whether the merger or a shard is to blame for a slow query.&lt;br/&gt;
&lt;br/&gt;
It only appears when the command is an aggregate or a getMore on an aggregation cursor.  It only appears on the merging node.</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16942"><![CDATA[Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>PM-1639</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            3 years, 21 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_16465" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Linked BF Score</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>35.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>bernard.gorman@mongodb.com</customfieldvalue>
            <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>david.percy@mongodb.com</customfieldvalue>
            <customfieldvalue>david.storch@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrjrkf:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr5orr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="2752">Query 2019-02-11</customfieldvalue>
    <customfieldvalue id="2841">Query 2019-05-06</customfieldvalue>
    <customfieldvalue id="3284">Query 2019-11-04</customfieldvalue>
    <customfieldvalue id="3285">Query 2019-11-18</customfieldvalue>
    <customfieldvalue id="3288">Query 2019-12-30</customfieldvalue>
    <customfieldvalue id="3289">Query 2020-01-13</customfieldvalue>
    <customfieldvalue id="3290">Query 2020-01-27</customfieldvalue>
    <customfieldvalue id="3291">Query 2020-02-10</customfieldvalue>
    <customfieldvalue id="3292">Query 2020-02-24</customfieldvalue>
    <customfieldvalue id="3494">Query 2020-03-09</customfieldvalue>
    <customfieldvalue id="3496">Query 2020-03-23</customfieldvalue>
    <customfieldvalue id="3497">Query 2020-04-06</customfieldvalue>
    <customfieldvalue id="3498">Query 2020-04-20</customfieldvalue>
    <customfieldvalue id="3499">Query 2020-05-04</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_17051" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Teams Impacted</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16943"><![CDATA[Cloud]]></customfieldvalue>
    <customfieldvalue key="16944"><![CDATA[Docs]]></customfieldvalue>
    <customfieldvalue key="16945"><![CDATA[Drivers]]></customfieldvalue>
    <customfieldvalue key="16946"><![CDATA[Triage and Release]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htggqv:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>