<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:54:01 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-40100] Mongos hangs without any log messages</title>
                <link>https://jira.mongodb.org/browse/SERVER-40100</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Symptoms: mongos listen port, but don&apos;t serve client requests for few hours (until restart). Nothing suspicious in the logs found.&lt;/p&gt;

&lt;p&gt;I was make core dump and backtrace for all threads (backtrace file is attached to this issue).&lt;/p&gt;

&lt;p&gt;How often: around twice a month per production environment.&lt;/p&gt;

&lt;p&gt;Version:&lt;/p&gt;

&lt;p&gt;&lt;tt&gt;mongos version v4.0.4&lt;/tt&gt;&lt;br/&gt;
{{ git version: f288a3bdf201007f3693c58e140056adf8b04839}}&lt;br/&gt;
{{ OpenSSL version: OpenSSL 1.0.2g 1 Mar 2016}}&lt;br/&gt;
{{ allocator: tcmalloc}}&lt;br/&gt;
{{ modules: none}}&lt;br/&gt;
{{ build environment:}}&lt;br/&gt;
{{ distmod: ubuntu1604}}&lt;br/&gt;
{{ distarch: x86_64}}&lt;br/&gt;
{{ target_arch: x86_64}}&lt;/p&gt;

&lt;p&gt;Topology:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;single shard cluster&lt;/li&gt;
	&lt;li&gt;3-nodes configuration replicaset;&lt;/li&gt;
	&lt;li&gt;4-nodes shard replicaset;&lt;/li&gt;
	&lt;li&gt;mongos instance on each client host.&lt;/li&gt;
&lt;/ul&gt;
</description>
                <environment></environment>
        <key id="715960">SERVER-40100</key>
            <summary>Mongos hangs without any log messages</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="3">Duplicate</resolution>
                                        <assignee username="eric.sedor@mongodb.com">Eric Sedor</assignee>
                                    <reporter username="bozaro">Artem</reporter>
                        <labels>
                    </labels>
                <created>Wed, 13 Mar 2019 08:20:03 +0000</created>
                <updated>Mon, 8 Jan 2024 15:22:54 +0000</updated>
                            <resolved>Mon, 15 Apr 2019 22:15:57 +0000</resolved>
                                    <version>4.0.4</version>
                                                    <component>Internal Code</component>
                                        <votes>0</votes>
                                    <watches>12</watches>
                                                                                                                <comments>
                            <comment id="2238199" author="bozaro" created="Wed, 8 May 2019 06:42:03 +0000"  >&lt;p&gt;After removing queries, killed by `maxTimeMS` timeout we had no any hangs.&lt;/p&gt;</comment>
                            <comment id="2214377" author="eric.sedor" created="Mon, 15 Apr 2019 22:14:15 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bozaro&quot; class=&quot;user-hover&quot; rel=&quot;bozaro&quot;&gt;bozaro&lt;/a&gt;, we believe &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-40110&quot; title=&quot;ClusterCursorManager::CursorEntry::isKillPending() should not call checkForInterrupt&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-40110&quot;&gt;&lt;del&gt;SERVER-40110&lt;/del&gt;&lt;/a&gt; will address this issue. Can you please watch that ticket for updates? Thank you again for your report.&lt;/p&gt;</comment>
                            <comment id="2185465" author="eric.sedor" created="Tue, 19 Mar 2019 22:47:21 +0000"  >&lt;p&gt;Thanks &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bozaro&quot; class=&quot;user-hover&quot; rel=&quot;bozaro&quot;&gt;bozaro&lt;/a&gt;, this is good to know. We are attempting an internal reproduction around lots of queries with $maxTimeMS set.&lt;/p&gt;</comment>
                            <comment id="2184678" author="bozaro" created="Tue, 19 Mar 2019 12:42:47 +0000"  >&lt;p&gt;Looks like we fix query timeouts.&lt;/p&gt;

&lt;p&gt;Problem query looks like: `db.someCollection.find({_id: {$in: &lt;span class=&quot;error&quot;&gt;&amp;#91;... over 20 000 values ...&amp;#93;&lt;/span&gt;}}).maxTimeMS(10000)`. And there are all of requested `_id` in database.&lt;/p&gt;</comment>
                            <comment id="2184592" author="bozaro" created="Tue, 19 Mar 2019 07:11:00 +0000"  >&lt;p&gt;The incident with the last backtraces occurred around&#160;`2019-03-18T18:47:44.016+0000Z`.&lt;/p&gt;

&lt;p&gt;About maxTimeMS: we use this option for almost all queries with timeout 10 sec.&lt;/p&gt;

&lt;p&gt;Also, we have increased the timeouts count in recent times due to some features. We are trying to reduce it.&#160;It is very likely that this may be related.&lt;/p&gt;</comment>
                            <comment id="2184391" author="eric.sedor" created="Mon, 18 Mar 2019 21:25:07 +0000"  >&lt;p&gt;Thanks for the additional information &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bozaro&quot; class=&quot;user-hover&quot; rel=&quot;bozaro&quot;&gt;bozaro&lt;/a&gt;; we will be attempting an internal reproduction of this.&lt;/p&gt;

&lt;p&gt;Can you clarify what time UTC that latest &quot;similar&quot; issue occurred?&lt;/p&gt;

&lt;p&gt;In case it helps us, can you let us know if you are setting &lt;a href=&quot;https://docs.mongodb.com/manual/reference/operator/meta/maxTimeMS/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;maxTimeMS&lt;/a&gt; for operations, and if so what value you&apos;re providing?&lt;/p&gt;</comment>
                            <comment id="2182494" author="bozaro" created="Sun, 17 Mar 2019 12:40:01 +0000"  >&lt;p&gt;Today we got very simillar issue on one host.&lt;/p&gt;

&lt;p&gt;Backtraces and diagnostics data was attached.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;</comment>
                            <comment id="2179927" author="jason.carey" created="Wed, 13 Mar 2019 18:11:59 +0000"  >&lt;p&gt;My first theory: in ClusterCursorManager::stats, and specifically ClusterCursorManager::CursorEntry::isKillPending().  See thread 11:&lt;/p&gt;

&lt;blockquote&gt;
&lt;p&gt;Thread 11 (Thread 0x7fd2289c6700 (LWP 40703)):&lt;br/&gt;
#0  __lll_lock_wait () at ../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135&lt;br/&gt;
#1  0x00007fd22d9bcdbd in _&lt;em&gt;GI&lt;/em&gt;_&lt;em&gt;pthread_mutex_lock (mutex=0x5611e57b2d00) at ../nptl/pthread&lt;/em&gt;&amp;gt;&lt;br/&gt;
#2  0x00005611e3b1c3c9 in mongo::OperationContext::markKilled(mongo::ErrorCodes::Error) ()&lt;br/&gt;
#3  0x00005611e3b1c968 in mongo::OperationContext::checkForInterruptNoAssert() ()&lt;br/&gt;
#4  0x00005611e33bb118 in mongo::ClusterCursorManager::stats() const ()&lt;br/&gt;
#5  0x00005611e2fc7d0f in mongo::(anonymous namespace)::ClusterCursorStats::appendAtLeaf(mongo&amp;gt;&lt;br/&gt;
#6  0x00005611e36aeb4d in mongo::MetricTree::appendTo(mongo::BSONObjBuilder&amp;amp;) const ()&lt;br/&gt;
#7  0x00005611e36aecd6 in mongo::MetricTree::appendTo(mongo::BSONObjBuilder&amp;amp;) const ()&lt;br/&gt;
#8  0x00005611e35fcc94 in mongo::CmdServerStatus::run(mongo::OperationContext*, std::__cxx11::&amp;gt;&lt;br/&gt;
#9  0x00005611e3607b4d in mongo::CommandHelpers::runCommandDirectly(mongo::OperationContext*, &amp;gt;&lt;br/&gt;
#10 0x00005611e3309bcd in mongo::FTDCSimpleInternalCommandCollector::collect(mongo::OperationC&amp;gt;&lt;br/&gt;
#11 0x00005611e33193f5 in mongo::FTDCCollectorCollection::collect(mongo::Client*) ()&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Note that we&apos;re in checkForInterruptNoAssert from ClusterCusorManager::stats(), which in turn is calling markKilled.&lt;/p&gt;

&lt;p&gt;For why, see &lt;a href=&quot;https://github.com/mongodb/mongo/blob/091b3c02c03d59db32809ade8264e163adc12cd2/src/mongo/s/query/cluster_cursor_manager.h#L544-L549&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;isKillPending()&lt;/a&gt;&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;          &lt;/span&gt;&lt;span style=&quot;color: #006699; font-weight: bold; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;bool&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; isKillPending() &lt;/span&gt;&lt;span style=&quot;color: #006699; font-weight: bold; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;const&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;              &lt;/span&gt;&lt;span style=&quot;color: #008200; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;// A cursor is kill pending if it&apos;s checked out by an OperationContext that was&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;              &lt;/span&gt;&lt;span style=&quot;color: #008200; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;// interrupted.&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;              &lt;/span&gt;&lt;span style=&quot;color: #006699; font-weight: bold; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;return&lt;/span&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; _operationUsingCursor &amp;amp;&amp;amp;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;                  !_operationUsingCursor-&amp;gt;checkForInterruptNoAssert().isOK();&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;          }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;That&apos;s a problem because:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;the rule for opCtx is that the owning thread can call all methods, and external threads can call some methods, while holding the client lock
	&lt;ul&gt;
		&lt;li&gt;ftdc isn&apos;t holding the client lock&lt;/li&gt;
	&lt;/ul&gt;
	&lt;/li&gt;
	&lt;li&gt;checkForInterruptNoAssert isn&apos;t meant to be called from off thread.  opCtx-&amp;gt;isKillPending() is the way to do that&lt;/li&gt;
	&lt;li&gt;checkForInterrupt can actually invoke markKilled, if the op is now timed out&lt;/li&gt;
	&lt;li&gt;markKilled does a dance with waitForConditionOrInterrupt that, when the latter is active, does:
	&lt;ul&gt;
		&lt;li&gt;an unlock of the client lock.  This probably kicks the system into an unanticipated state&lt;/li&gt;
		&lt;li&gt;a lock of the mutex specified in waitForConditionOrInterrupt&lt;/li&gt;
		&lt;li&gt;a lock of the client lock&lt;/li&gt;
		&lt;li&gt;Setting the killcode&lt;/li&gt;
		&lt;li&gt;unlocking the mutex specified in waitForConditionOrInterrupt&lt;/li&gt;
	&lt;/ul&gt;
	&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;I think this doesn&apos;t usually come up because the ops using cluster cursors usually haven&apos;t exceeded maxtimems.  That may be an avenue for an isolated repro.&lt;/p&gt;
</comment>
                            <comment id="2179465" author="eric.sedor" created="Wed, 13 Mar 2019 14:27:54 +0000"  >&lt;p&gt;Hello and thanks for the report and backtrace. Can you please also provide the logs leading up to a hang as well as an archive (tar or zip) the &lt;tt&gt;$dbpath/diagnostic.data&lt;/tt&gt; directory (described &lt;a href=&quot;https://docs.mongodb.com/manual/administration/analyzing-mongodb-performance/#full-time-diagnostic-data-capture&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;here&lt;/a&gt;)?&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                            <outwardlinks description="duplicates">
                                        <issuelink>
            <issuekey id="716349">SERVER-40110</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="716349">SERVER-40110</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="211247" name="bt.txt" size="3388733" author="bozaro" created="Wed, 13 Mar 2019 08:16:10 +0000"/>
                            <attachment id="211603" name="mongos.20190317.bt.txt" size="182350" author="bozaro" created="Sun, 17 Mar 2019 12:29:38 +0000"/>
                            <attachment id="211604" name="mongos.20190317.diagnostics.tgz" size="2224260" author="bozaro" created="Sun, 17 Mar 2019 12:29:38 +0000"/>
                            <attachment id="211688" name="mongos.20190318.bt.txt" size="3240319" author="bozaro" created="Mon, 18 Mar 2019 20:22:06 +0000"/>
                            <attachment id="211689" name="mongos.20190318.diagnostic.tgz" size="9598818" author="bozaro" created="Mon, 18 Mar 2019 20:22:07 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 13 Mar 2019 14:27:54 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        4 years, 40 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            4 years, 40 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>bozaro</customfieldvalue>
            <customfieldvalue>eric.sedor@mongodb.com</customfieldvalue>
            <customfieldvalue>mira.carey@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|huqnz3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hugb07:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10750" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Steps To Reproduce</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>&lt;p&gt;Unknown.&lt;/p&gt;</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[eric.sedor@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|huqa8f:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>