<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:17:51 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-48695] setAppliedThrough must set orderedCommit=false on the recovery unit</title>
                <link>https://jira.mongodb.org/browse/SERVER-48695</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;When we do a timestamped write that may not be strictly ordered with respect to the oplog (that is, if it has a timestamp less than or equal to the latest optime in the oplog), we must set orderedCommit to false to ensure oplog visibility is updated.  ReplicationConsistencyMarkers::setAppliedThrough does not do this.&lt;/p&gt;

&lt;p&gt;It might make sense to do this in StorageInterfaceImpl::_updateWithQuery to catch any others.&lt;/p&gt;</description>
                <environment></environment>
        <key id="1377217">SERVER-48695</key>
            <summary>setAppliedThrough must set orderedCommit=false on the recovery unit</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="matthew.russotto@mongodb.com">Matthew Russotto</assignee>
                                    <reporter username="matthew.russotto@mongodb.com">Matthew Russotto</reporter>
                        <labels>
                    </labels>
                <created>Wed, 10 Jun 2020 17:06:26 +0000</created>
                <updated>Sun, 29 Oct 2023 22:07:13 +0000</updated>
                            <resolved>Thu, 11 Jun 2020 15:01:10 +0000</resolved>
                                                    <fixVersion>4.0.20</fixVersion>
                    <fixVersion>4.4.0-rc10</fixVersion>
                    <fixVersion>4.2.9</fixVersion>
                    <fixVersion>4.7.0</fixVersion>
                                    <component>Replication</component>
                    <component>Storage</component>
                                        <votes>0</votes>
                                    <watches>4</watches>
                                                                                                                <comments>
                            <comment id="3219525" author="xgen-internal-githook" created="Tue, 23 Jun 2020 14:31:56 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Matthew Russotto&apos;, &apos;email&apos;: &apos;matthew.russotto@10gen.com&apos;, &apos;username&apos;: &apos;mtrussotto&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-48695&quot; title=&quot;setAppliedThrough must set orderedCommit=false on the recovery unit&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-48695&quot;&gt;&lt;del&gt;SERVER-48695&lt;/del&gt;&lt;/a&gt; setAppliedThrough must set orderedCommit=false on the recovery unit&lt;/p&gt;

&lt;p&gt;(cherry picked from commit e52733a000b23cde7dcd2b2fc765a8a1f1012165)&lt;br/&gt;
Branch: v4.2&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/26628142b33deded802f1635990915d55970a48d&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/26628142b33deded802f1635990915d55970a48d&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3219457" author="xgen-internal-githook" created="Tue, 23 Jun 2020 14:09:57 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Matthew Russotto&apos;, &apos;email&apos;: &apos;matthew.russotto@10gen.com&apos;, &apos;username&apos;: &apos;mtrussotto&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-48695&quot; title=&quot;setAppliedThrough must set orderedCommit=false on the recovery unit&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-48695&quot;&gt;&lt;del&gt;SERVER-48695&lt;/del&gt;&lt;/a&gt; setAppliedThrough must set orderedCommit=false on the recovery unit&lt;/p&gt;

&lt;p&gt;(cherry picked from commit e52733a000b23cde7dcd2b2fc765a8a1f1012165)&lt;br/&gt;
Branch: v4.0&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/3da338ad27cb52475f48696bd7d88ee55576c4dc&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/3da338ad27cb52475f48696bd7d88ee55576c4dc&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3212491" author="xgen-internal-githook" created="Wed, 17 Jun 2020 16:10:05 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Matthew Russotto&apos;, &apos;email&apos;: &apos;matthew.russotto@10gen.com&apos;, &apos;username&apos;: &apos;mtrussotto&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-48695&quot; title=&quot;setAppliedThrough must set orderedCommit=false on the recovery unit&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-48695&quot;&gt;&lt;del&gt;SERVER-48695&lt;/del&gt;&lt;/a&gt; setAppliedThrough must set orderedCommit=false on the recovery unit&lt;/p&gt;

&lt;p&gt;(cherry picked from commit e52733a000b23cde7dcd2b2fc765a8a1f1012165)&lt;br/&gt;
Branch: v4.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/8f4ebc760ca191898c5405493b2663eaa125c57a&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/8f4ebc760ca191898c5405493b2663eaa125c57a&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3205176" author="xgen-internal-githook" created="Thu, 11 Jun 2020 14:53:04 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Matthew Russotto&apos;, &apos;email&apos;: &apos;matthew.russotto@10gen.com&apos;, &apos;username&apos;: &apos;mtrussotto&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-48695&quot; title=&quot;setAppliedThrough must set orderedCommit=false on the recovery unit&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-48695&quot;&gt;&lt;del&gt;SERVER-48695&lt;/del&gt;&lt;/a&gt; setAppliedThrough must set orderedCommit=false on the recovery unit&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/e52733a000b23cde7dcd2b2fc765a8a1f1012165&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/e52733a000b23cde7dcd2b2fc765a8a1f1012165&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3205098" author="daniel.gottlieb@10gen.com" created="Thu, 11 Jun 2020 14:25:50 +0000"  >&lt;p&gt;Thanks &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=matthew.russotto&quot; class=&quot;user-hover&quot; rel=&quot;matthew.russotto&quot;&gt;matthew.russotto&lt;/a&gt;, that makes sense. Delaying the oplog manager thread might have been necessary. I forgot the fact that the manager will never move backwards, so if the last hole was filled and advanced the oplog read timestamp, it&apos;s too late to make the hang happen. It&apos;s not sufficient to just hold back the &lt;tt&gt;all_durable&lt;/tt&gt; timestamp.&lt;/p&gt;</comment>
                            <comment id="3204838" author="matthew.russotto" created="Thu, 11 Jun 2020 12:24:02 +0000"  >&lt;p&gt;I think opening large windows may not have worked because in this case, the transactions don&apos;t overlap.  We need to get the RSTL in X mode before stepping down, which means the primary writes are done by the time the problem write appears.  The sequence must be&lt;/p&gt;

&lt;p&gt;Primary write with optime X completes.  all_durable is now X.&lt;br/&gt;
Visibility CV  is triggered.&lt;br/&gt;
The visibility thread reaches somewhere here &lt;tt&gt;&lt;a href=&quot;http://morningstar/mongodb_v4.4/source/src/mongo/db/storage/wiredtiger/wiredtiger_oplog_manager.cpp#191-224&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://morningstar/mongodb_v4.4/source/src/mongo/db/storage/wiredtiger/wiredtiger_oplog_manager.cpp#191-224&lt;/a&gt;&lt;/tt&gt;&lt;br/&gt;
The orderedCommit=true write starts with optime X.  all_durable is now X-1&lt;br/&gt;
The visibility thread reads all_durable &lt;tt&gt;&lt;a href=&quot;http://morningstar/mongodb_v4.4/source/src/mongo/db/storage/wiredtiger/wiredtiger_oplog_manager.cpp#228&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://morningstar/mongodb_v4.4/source/src/mongo/db/storage/wiredtiger/wiredtiger_oplog_manager.cpp#228&lt;/a&gt;&lt;/tt&gt;&lt;br/&gt;
The orderedCommit=true write completes.  all_durable is now X, but no trigger so we&apos;re stuck.&lt;/p&gt;

&lt;p&gt;Opening the window on the problem write makes this more likely to happen, but the problem write still has to start within ~1ms of the last primary write (less if someone else is checking visibility, e.g. some secondary oplog reader), or it&apos;s too late.&lt;/p&gt;</comment>
                            <comment id="3204342" author="daniel.gottlieb@10gen.com" created="Thu, 11 Jun 2020 01:46:35 +0000"  >&lt;p&gt;I went back to my patch; the invariant I set did capture this usage as potentially problematic. The problem was that there were quite a few replication calls that set a timestamp without setting &lt;tt&gt;orderedCommit(false)&lt;/tt&gt;. I was unable to correctly verify whether a timestamp, &quot;ordered&quot; transaction usage was benign or not.&lt;/p&gt;

&lt;p&gt;That said, I tried another approach that removed my verification and instead opened large windows on anything that was potentially problematic to allow a the oplog manager to see the hole. But that strategy relied on other writes that do trigger oplog visibility to be drained by (this problem write had to be the last thing to commit). My hypothesis is that there must be other asynchronous oplog visibility triggering writers in the system that nullified my attempt to open a window.&lt;/p&gt;</comment>
                            <comment id="3204249" author="daniel.gottlieb@10gen.com" created="Wed, 10 Jun 2020 23:38:45 +0000"  >&lt;blockquote&gt;
&lt;p&gt;Could the oplog no-holes point be determined by the oldest oplog timestamp in progress - 1, or last oplog timestamp written if none is in progress? &lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;That&apos;s what I was suggesting with:&lt;/p&gt;
&lt;blockquote&gt;
&lt;p&gt;I think a more holistic fix would be going back to having MongoDB manage holes.&lt;/p&gt;&lt;/blockquote&gt;

&lt;blockquote&gt;
&lt;p&gt;If so maybe we could do it inline without an extra thread.&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;That was actually a goal of Replicate before Journaling, but it was unattained. I&apos;m unsure what the challenge was.&lt;/p&gt;</comment>
                            <comment id="3204213" author="matthew.russotto" created="Wed, 10 Jun 2020 22:59:09 +0000"  >&lt;p&gt;Could the oplog no-holes point be determined by the oldest oplog timestamp in progress - 1, or last oplog timestamp written if none is in progress?  (with complications for rollback)?  If so maybe we could do it inline without an extra thread.  But that would be a major change so we&apos;d still need this for backport.&lt;/p&gt;</comment>
                            <comment id="3203838" author="daniel.gottlieb@10gen.com" created="Wed, 10 Jun 2020 19:25:12 +0000"  >&lt;p&gt;Alternatively, we can flip the &lt;a href=&quot;https://github.com/mongodb/mongo/blob/0987c120f552ab6d347f6b1b6574345e8c938c32/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h#L257&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;ordered commit&lt;/a&gt; default to false and have things opt-out of triggering visibility (e.g: secondaries writing oplog entries).&lt;/p&gt;

&lt;p&gt;It&apos;s also less clear the performance impact of having ordered commit. With replicate before journal, triggering visibility no longer does a journal flush.&lt;/p&gt;</comment>
                            <comment id="3203736" author="louis.williams" created="Wed, 10 Jun 2020 18:38:09 +0000"  >&lt;p&gt;I attempted to run a &lt;a href=&quot;https://evergreen.mongodb.com/version/5ec55fdea4cf47231fbb8040&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;patch build&lt;/a&gt; a few months ago that invariants all timestamps must be assigned in increasing order, or orderedCommit must be false. I had to explicitly set orderedCommit=false all over the place and there&apos;s still some redness. One of the places I had to change was in setAppliedThrough. We could potentially expand upon that invariant since it caught one bug, &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-48565&quot; title=&quot;Ghost timestamps on primaries can break causality in multi-document &amp;quot;snapshot&amp;quot; reads &quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-48565&quot;&gt;&lt;del&gt;SERVER-48565&lt;/del&gt;&lt;/a&gt;, and would have caught this had I noticed the problem at the time.&lt;/p&gt;</comment>
                            <comment id="3203714" author="daniel.gottlieb@10gen.com" created="Wed, 10 Jun 2020 18:28:21 +0000"  >&lt;p&gt;I think a more holistic fix would be going back to having MongoDB manage holes. The problem with WT is that it can&apos;t tell whether a transaction setting a timestamp is actually related to an oplog hole.&lt;/p&gt;

&lt;p&gt;I also don&apos;t believe we need the &lt;tt&gt;waitForAllWritesToBeVisible&lt;/tt&gt; check there. IIRC, that was added because the &quot;truncate oplog&quot; call was &lt;a href=&quot;https://github.com/mongodb/mongo/blob/master/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp#L1998&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;subject to visibility&lt;/a&gt; (implicitly/unintentionally), and &lt;tt&gt;cappedTruncateAfter&lt;/tt&gt; wouldn&apos;t delete the documents the that weren&apos;t visible to it! I think we changed that behavior (not that I see where/how). I firmly believe calling &lt;tt&gt;cappedTruncateAfter&lt;/tt&gt; should be not subject to visibility requirements.&lt;/p&gt;

&lt;p&gt;When we did this visibility stuff for 3.6, the usage of timestamps was small and limited to things that actually touched the oplog. That&apos;s not the case now. We should rethink having WT do the tracking.&lt;/p&gt;

&lt;p&gt;But the fix in this ticket is obviously easier and less risky....&lt;/p&gt;</comment>
                            <comment id="3203698" author="matthew.russotto" created="Wed, 10 Jun 2020 18:18:55 +0000"  >&lt;p&gt;We&apos;d have to have some way of whitelisting the ones which are allowed to be orderedCommit.   &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=daniel.gottlieb&quot; class=&quot;user-hover&quot; rel=&quot;daniel.gottlieb&quot;&gt;daniel.gottlieb&lt;/a&gt; tried to write one but it didn&apos;t catch this for some reason.&lt;/p&gt;</comment>
                            <comment id="3203664" author="judah.schvimer" created="Wed, 10 Jun 2020 17:56:51 +0000"  >&lt;p&gt;Is there an invariant we could add to catch this?&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>14.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="18953"><![CDATA[v4.4]]></customfieldvalue>
    <customfieldvalue key="16775"><![CDATA[v4.2]]></customfieldvalue>
    <customfieldvalue key="15640"><![CDATA[v4.0]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 10 Jun 2020 17:56:51 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        3 years, 33 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            3 years, 33 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_16465" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Linked BF Score</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>20.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>daniel.gottlieb@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>judah.schvimer@mongodb.com</customfieldvalue>
            <customfieldvalue>louis.williams@mongodb.com</customfieldvalue>
            <customfieldvalue>matthew.russotto@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hxpci7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hxcft3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hxoyrj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>