<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:48:58 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-38447] create index blocked  by checkpoint mutex</title>
                <link>https://jira.mongodb.org/browse/SERVER-38447</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;when create index with background on one database, the database x lock held for a long time, with the help of gdb, i found it was blocked by the wiredtiger&apos;s checkpoint mutex, the gdb stack is:&lt;/p&gt;

&lt;p&gt;Thread 145 (Thread 0x7fab66ff4700 (LWP 1426590)):&lt;br/&gt;
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135&lt;br/&gt;
#1 0x00007fab8e9cc479 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0&lt;br/&gt;
#2 0x00007fab8e9cc2a0 in _&lt;em&gt;GI&lt;/em&gt;__pthread_mutex_lock (mutex=0x55620d9a7538) at ../nptl/pthread_mutex_lock.c:79&lt;br/&gt;
#3 0x0000556208d4180b in __session_alter ()&lt;br/&gt;
#4 0x0000556208cdb5c5 in mongo::WiredTigerUtil::setTableLogging(_&lt;em&gt;wt_session*, std::&lt;/em&gt;_cxx11::basic_string&amp;lt;char, std::char_traits&amp;lt;char&amp;gt;, std::allocator&amp;lt;char&amp;gt; &amp;gt; const&amp;amp;, bo&lt;br/&gt;
ol) ()&lt;br/&gt;
#5 0x0000556208cdbb58 in mongo::WiredTigerUtil::setTableLogging(mongo::OperationContext*, std::__cxx11::basic_string&amp;lt;char, std::char_traits&amp;lt;char&amp;gt;, std::allocator&amp;lt;char&amp;gt; &amp;gt;&lt;br/&gt;
 const&amp;amp;, bool) ()&lt;br/&gt;
#6 0x0000556208ca6ea4 in mongo::WiredTigerIndex::WiredTigerIndex(mongo::OperationContext*, std::__cxx11::basic_string&amp;lt;char, std::char_traits&amp;lt;char&amp;gt;, std::allocator&amp;lt;char&amp;gt;&lt;br/&gt;
&amp;gt; const&amp;amp;, mongo::IndexDescriptor const*, mongo::KVPrefix, bool) ()&lt;br/&gt;
#7 0x0000556208ca7125 in mongo::WiredTigerIndexStandard::WiredTigerIndexStandard(mongo::OperationContext*, std::__cxx11::basic_string&amp;lt;char, std::char_traits&amp;lt;char&amp;gt;, std::&lt;br/&gt;
allocator&amp;lt;char&amp;gt; &amp;gt; const&amp;amp;, mongo::IndexDescriptor const*, mongo::KVPrefix, bool) ()&lt;br/&gt;
#8 0x0000556208cae9a7 in mongo::WiredTigerKVEngine::getGroupedSortedDataInterface(mongo::OperationContext*, mongo::StringData, mongo::IndexDescriptor const*, mongo::KVPr&lt;br/&gt;
efix) ()&lt;br/&gt;
#9 0x0000556208e2ab4d in mongo::KVDatabaseCatalogEntry::getIndex(mongo::OperationContext*, mongo::CollectionCatalogEntry const*, mongo::IndexCatalogEntry*) ()&lt;br/&gt;
#10 0x000055620936b550 in mongo::IndexCatalogImpl::_setupInMemoryStructures(mongo::OperationContext*, std::unique_ptr&amp;lt;mongo::IndexDescriptor, std::default_delete&amp;lt;mongo::I&lt;br/&gt;
ndexDescriptor&amp;gt; &amp;gt;, bool) ()&lt;br/&gt;
#11 0x0000556209d21a9e in mongo::IndexCatalog::_setupInMemoryStructures(mongo::OperationContext*, std::unique_ptr&amp;lt;mongo::IndexDescriptor, std::default_delete&amp;lt;mongo::Index&lt;br/&gt;
Descriptor&amp;gt; &amp;gt;, bool) ()&lt;br/&gt;
#12 0x0000556209364029 in mongo::IndexCatalogImpl::IndexBuildBlock::init() ()&lt;br/&gt;
#13 0x0000556209374b2e in mongo::MultiIndexBlockImpl::init(std::vector&amp;lt;mongo::BSONObj, std::allocator&amp;lt;mongo::BSONObj&amp;gt; &amp;gt; const&amp;amp;) ()&lt;br/&gt;
#14 0x000055620925cb0f in auto mongo::writeConflictRetry&amp;lt;mongo::CmdCreateIndex::errmsgRun(mongo::OperationContext*, std::_&lt;em&gt;cxx11::basic_string&amp;lt;char, std::char_traits&amp;lt;char&amp;gt;, std::allocator&amp;lt;char&amp;gt; &amp;gt; const&amp;amp;, mongo::BSONObj const&amp;amp;, std::&lt;/em&gt;&lt;em&gt;cxx11::basic_string&amp;lt;char, std::char_traits&amp;lt;char&amp;gt;, std::allocator&amp;lt;char&amp;gt; &amp;gt;&amp;amp;, mongo::BSONObjBuilder&amp;amp;)::{lambda()#6}&amp;gt;(mongo::OperationContext*, mongo::StringData, mongo::CmdCreateIndex::errmsgRun(mongo::OperationContext*, std::&lt;/em&gt;&lt;em&gt;cxx11::basic_string&amp;lt;char, std::char_traits&amp;lt;char&amp;gt;, std::allocator&amp;lt;char&amp;gt; &amp;gt; const&amp;amp;, mongo::BSONObj const&amp;amp;, std::&lt;/em&gt;&lt;em&gt;cxx11::basic_string&amp;lt;char, std::char_traits&amp;lt;char&amp;gt;, std::allocator&amp;lt;char&amp;gt; &amp;gt;&amp;amp;, mongo::BSONObjBuilder&amp;amp;)::{lambda()#6}, mongo::CmdCreateIndex::errmsgRun(mongo::OperationContext*, std::&lt;/em&gt;&lt;em&gt;cxx11::basic_string&amp;lt;char, std::char_traits&amp;lt;char&amp;gt;, std::allocator&amp;lt;char&amp;gt; &amp;gt; const&amp;amp;, mongo::BSONObj const&amp;amp;, std::&lt;/em&gt;_cxx11::basic_string&amp;lt;char, std::char_traits&amp;lt;char&amp;gt;, std::allocator&amp;lt;char&amp;gt; &amp;gt;&amp;amp;, mongo::BSONObjBuilder&amp;amp;)::{lambda()#6}&amp;amp;&amp;amp;) ()&lt;/p&gt;</description>
                <environment></environment>
        <key id="646517">SERVER-38447</key>
            <summary>create index blocked  by checkpoint mutex</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="3">Duplicate</resolution>
                                        <assignee username="backlog-server-triage">Backlog - Triage Team</assignee>
                                    <reporter username="hustchensi">chensi</reporter>
                        <labels>
                    </labels>
                <created>Fri, 7 Dec 2018 03:24:48 +0000</created>
                <updated>Fri, 7 Apr 2023 11:43:11 +0000</updated>
                            <resolved>Mon, 1 Nov 2021 19:54:29 +0000</resolved>
                                    <version>4.0.3</version>
                                                    <component>Concurrency</component>
                                        <votes>1</votes>
                                    <watches>15</watches>
                                                                                                                <comments>
                            <comment id="4161622" author="thomas.schubert" created="Mon, 1 Nov 2021 19:54:15 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=chensi_04%40126.com&quot; class=&quot;user-hover&quot; rel=&quot;chensi_04@126.com&quot;&gt;chensi_04@126.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;I believe this issue has been resolved by &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-37484&quot; title=&quot;Only alter table logging settings on startup and not collection creation&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-37484&quot;&gt;&lt;del&gt;SERVER-37484&lt;/del&gt;&lt;/a&gt;. Would you please consider upgrading and let us know if the issue is resolved?&lt;/p&gt;

&lt;p&gt;Thank you,&lt;br/&gt;
Kelsey&lt;/p&gt;</comment>
                            <comment id="2100769" author="wolf_kdy" created="Thu, 27 Dec 2018 10:09:07 +0000"  >&lt;p&gt;@&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=daniel.hatcher&quot; class=&quot;user-hover&quot; rel=&quot;daniel.hatcher&quot;&gt;daniel.hatcher&lt;/a&gt;&#160;&lt;/p&gt;

&lt;p&gt;After a short-time conversation with Jack, we believe the problem has already been described with the stack in @&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=chensi_04%40126.com&quot; class=&quot;user-hover&quot; rel=&quot;chensi_04@126.com&quot;&gt;chensi_04@126.com&lt;/a&gt;&apos;s second post. Mean no offence but no more logs are needed for this problem.&lt;/p&gt;

&lt;p&gt;We can discuss about it with the latest mongo code base.&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;git rev-parse HEAD&lt;br/&gt;
 9ff8d7452ddf7370fc9047ebe9517fd7558914e6&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;1)in&#160;commands/create_indexes.cpp, line 311, target db&apos;s X lock is held.&lt;/p&gt;

&lt;p&gt;2)And in commmands/create_indexes.cpp, line&#160;383,&#160;indexer.init is called. If we track the call path, we can find&#160;&#160;WiredTigerUtil::setTableLogging is called,&lt;/p&gt;

&lt;p&gt;track deeper, we can find it gets into wiredtiger,&#160;session_api.c, line 670, the checkpoint mutex is taken.&lt;/p&gt;

&lt;p&gt;So, in conclusion, the db&apos;s X lock may block for quite a long time if a checkpoint happens to be doing. It may block for several seconds or tens of seconds if&lt;/p&gt;

&lt;p&gt;the mongod instance is suffering a heavy workload.&lt;/p&gt;

&lt;p&gt;It may or may not be a bug, but it is caused by the checkpoint mutex.&lt;/p&gt;

&lt;p&gt;And, what&apos;s important is that it&apos;s unacceptable for a background index building to block the whole db for quite a long time, because, it is a newly introduced&lt;/p&gt;

&lt;p&gt;problem. for the compatibilty with old versions, it should be treated as a bug.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;</comment>
                            <comment id="2086101" author="daniel.hatcher" created="Mon, 10 Dec 2018 19:53:26 +0000"  >&lt;p&gt;Hello Jack,&lt;/p&gt;

&lt;p&gt;Unfortunately, it will be very difficult for us to validate this issue unless we have the logs. I have generated a &lt;a href=&quot;https://10gen-httpsupload.s3.amazonaws.com/upload_forms/5c530eb8-5722-4d07-a3c7-22ef46e55786.html&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;Secure Upload Portal&lt;/a&gt; that you can use to upload files. Only MongoDB engineers can access the portal and files within are deleted after 180 days. Would this satisfy your security concerns?&lt;/p&gt;

&lt;p&gt;Thank you,&lt;/p&gt;

&lt;p&gt;Danny&lt;/p&gt;</comment>
                            <comment id="2084669" author="hustchensi" created="Sat, 8 Dec 2018 00:33:11 +0000"  >&lt;p&gt;1. I send the create index command, but &lt;br/&gt;
&#8220;build index on xxx.yyy&#8221; log was printed after twenty or thirty seconds. Before it printed, all requests on this database were blocked, because I saw many slow logs immediately after the log, all the slow logs indicate they are waiting the database lock.&lt;br/&gt;
Also I use poor man&#8217;s profiler found the gdb stack as above.&lt;/p&gt;

&lt;p&gt;2. On the primary &lt;br/&gt;
3. I mentioned in 1, but logs have some security, sorry about that&lt;br/&gt;
4. Later I will upload&lt;br/&gt;
&#21457;&#33258;&#25105;&#30340; iPhone&lt;/p&gt;
</comment>
                            <comment id="2084181" author="daniel.hatcher" created="Fri, 7 Dec 2018 18:38:27 +0000"  >&lt;p&gt;Hello Jack,&lt;/p&gt;

&lt;p&gt;I cannot validate what will happen if you manually change any code and we cannot troubleshoot any such changes. &lt;/p&gt;

&lt;p&gt;Can you please provide more context on the issue?&lt;br/&gt;
1. How did you determine that an inappropriate lock was held for a large amount of time?&lt;br/&gt;
2. Did this happen on the Primary or on a Secondary?&lt;br/&gt;
3. Please provide the &lt;tt&gt;mongod&lt;/tt&gt; log from the server.&lt;br/&gt;
4. Please provide the &quot;diagnostic.data&quot; folder located under the &lt;tt&gt;$dbpath&lt;/tt&gt;.&lt;/p&gt;

&lt;p&gt;Thank you,&lt;/p&gt;

&lt;p&gt;Danny&lt;/p&gt;</comment>
                            <comment id="2083539" author="hustchensi" created="Fri, 7 Dec 2018 04:56:21 +0000"  >&lt;p&gt;also, &#160;we need to fix this problem as soon as possible, is it ok if i just remove the setTableLogging from the following code:&lt;br/&gt;
WiredTigerIndex::WiredTigerIndex(OperationContext* ctx,&lt;br/&gt;
const std::string&amp;amp; uri,&lt;br/&gt;
const IndexDescriptor* desc,&lt;br/&gt;
KVPrefix prefix,&lt;br/&gt;
bool isReadOnly)&lt;br/&gt;
: _ordering(Ordering::make(desc-&amp;gt;keyPattern())),&lt;br/&gt;
_uri(uri),&lt;br/&gt;
_tableId(WiredTigerSession::genTableId()),&lt;br/&gt;
_collectionNamespace(desc-&amp;gt;parentNS()),&lt;br/&gt;
_indexName(desc-&amp;gt;indexName()),&lt;br/&gt;
_prefix(prefix),&lt;br/&gt;
_isIdIndex(desc-&amp;gt;isIdIndex()) {&lt;br/&gt;
auto version =WiredTigerUtil::checkApplicationMetadataFormatVersion(&lt;br/&gt;
ctx, uri, kMinimumIndexVersion, kMaximumIndexVersion);&lt;br/&gt;
if (!version.isOK()) {&lt;br/&gt;
Status versionStatus = version.getStatus();&lt;br/&gt;
Status indexVersionStatus(&lt;br/&gt;
ErrorCodes::UnsupportedFormat,&lt;br/&gt;
str::stream() &amp;lt;&amp;lt; versionStatus.reason() &amp;lt;&amp;lt;&quot; Index: &lt;/p&gt;
{name: &quot;&amp;lt;&amp;lt; desc-&amp;gt;indexName()
&amp;lt;&amp;lt;&quot;, ns: &quot;
&amp;lt;&amp;lt; desc-&amp;gt;parentNS()
&amp;lt;&amp;lt;&quot;}
&lt;p&gt; - version too new for this mongod.&quot;&lt;br/&gt;
&amp;lt;&amp;lt;&quot; See &lt;a href=&quot;http://dochub.mongodb.org/core/3.4-index-downgrade&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://dochub.mongodb.org/core/3.4-index-downgrade&lt;/a&gt; for detailed&quot;&lt;br/&gt;
&amp;lt;&amp;lt;&quot; instructions on how to handle this error.&quot;);&lt;br/&gt;
fassertFailedWithStatusNoTrace(28579, indexVersionStatus);&lt;br/&gt;
}&lt;br/&gt;
_dataFormatVersion = version.getValue();&lt;/p&gt;

&lt;p&gt;// Index data format 6 and 9 correspond to KeyString version V0 and data format 8 and 10&lt;br/&gt;
// correspond to KeyString version V1&lt;br/&gt;
_keyStringVersion = (_dataFormatVersion == kDataFormatV2KeyStringV1IndexVersionV2 ||&lt;br/&gt;
_dataFormatVersion == kDataFormatV4KeyStringV1UniqueIndexVersionV2)&lt;br/&gt;
? KeyString::Version::V1&lt;br/&gt;
: KeyString::Version::V0;&lt;/p&gt;

&lt;p&gt;&lt;ins&gt;&lt;em&gt;if (!isReadOnly) {&lt;/em&gt;&lt;/ins&gt;&lt;br/&gt;
&lt;ins&gt;&lt;em&gt;bool replicatedWrites =getGlobalReplSettings().usingReplSets() ||&lt;/em&gt;&lt;/ins&gt;&lt;br/&gt;
&lt;ins&gt;&lt;em&gt;repl::ReplSettings::shouldRecoverFromOplogAsStandalone();&lt;/em&gt;&lt;/ins&gt;&lt;br/&gt;
&lt;ins&gt;&lt;em&gt;uassertStatusOK(WiredTigerUtil::setTableLogging(&lt;/em&gt;&lt;/ins&gt;&lt;br/&gt;
&lt;ins&gt;&lt;em&gt;ctx,&lt;/em&gt;&lt;/ins&gt;&lt;br/&gt;
&lt;ins&gt;&lt;em&gt;uri,&lt;/em&gt;&lt;/ins&gt;&lt;br/&gt;
&lt;ins&gt;&lt;em&gt;WiredTigerUtil::useTableLogging(NamespaceString(desc-&amp;gt;parentNS()), replicatedWrites)));&lt;/em&gt;&lt;/ins&gt;&lt;br/&gt;
&lt;ins&gt;&lt;em&gt;}&lt;/em&gt;&lt;/ins&gt;&lt;br/&gt;
}&lt;br/&gt;
&#160;&lt;br/&gt;
thx&lt;/p&gt;</comment>
                            <comment id="2083517" author="hustchensi" created="Fri, 7 Dec 2018 03:26:23 +0000"  >&lt;p&gt;and, i see this: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-35625&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;https://jira.mongodb.org/browse/SERVER-35625&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;setTableLogging will be removed, but please tell the roadmap, thx.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                            <outwardlinks description="duplicates">
                                        <issuelink>
            <issuekey id="614603">SERVER-37484</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>7.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_12751" key="com.atlassian.jira.plugin.system.customfieldtypes:multiselect">
                        <customfieldname>Assigned Teams</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="25137"><![CDATA[Server Triage]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 7 Dec 2018 18:38:27 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        2 years, 14 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            2 years, 14 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>backlog-server-triage</customfieldvalue>
            <customfieldvalue>hustchensi</customfieldvalue>
            <customfieldvalue>daniel.hatcher@mongodb.com</customfieldvalue>
            <customfieldvalue>wolf_kdy</customfieldvalue>
            <customfieldvalue>kelsey.schubert@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|huf0fz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hu553b:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|huempb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>