<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:12:31 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-46818] Do untimestamped writes in IdempotencyTest</title>
                <link>https://jira.mongodb.org/browse/SERVER-46818</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;The unit test db_repl_test, more specifically: &lt;tt&gt;CheckUpdateSequencesAreIdempotent&lt;/tt&gt; writes timestamps on modifies that are out of order. Could the test call rollback to stable between iterations? Or does it have to be this way?&lt;/p&gt;

&lt;p&gt;For more detail see &lt;a href=&quot;https://jira.mongodb.org/browse/WT-5799&quot; title=&quot;Dont assume we have ordered timestamps when doing rec append original value&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-5799&quot;&gt;&lt;del&gt;WT-5799&lt;/del&gt;&lt;/a&gt;.&lt;/p&gt;</description>
                <environment></environment>
        <key id="1266955">SERVER-46818</key>
            <summary>Do untimestamped writes in IdempotencyTest</summary>
                <type id="3" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14718&amp;avatarType=issuetype">Task</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="lingzhi.deng@mongodb.com">Lingzhi Deng</assignee>
                                    <reporter username="luke.pearson@mongodb.com">Luke Pearson</reporter>
                        <labels>
                    </labels>
                <created>Thu, 12 Mar 2020 03:13:57 +0000</created>
                <updated>Sun, 29 Oct 2023 22:10:53 +0000</updated>
                            <resolved>Wed, 1 Apr 2020 22:03:01 +0000</resolved>
                                                    <fixVersion>4.4.0-rc0</fixVersion>
                    <fixVersion>4.7.0</fixVersion>
                                    <component>Replication</component>
                                        <votes>0</votes>
                                    <watches>7</watches>
                                                                                                                <comments>
                            <comment id="3031660" author="xgen-internal-githook" created="Wed, 8 Apr 2020 18:27:01 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Lingzhi Deng&apos;, &apos;email&apos;: &apos;lingzhi.deng@mongodb.com&apos;, &apos;username&apos;: &apos;ldennis&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-46818&quot; title=&quot;Do untimestamped writes in IdempotencyTest&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-46818&quot;&gt;&lt;del&gt;SERVER-46818&lt;/del&gt;&lt;/a&gt;: Do untimestamped writes in idemptency tests&lt;/p&gt;

&lt;p&gt;(cherry picked from commit 53f344385322f7e9a93779e4a515167e766182cc)&lt;br/&gt;
Branch: v4.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/66a0bfe5c1afa5c73d2ce74ebe3eeef26703cb38&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/66a0bfe5c1afa5c73d2ce74ebe3eeef26703cb38&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3022054" author="xgen-internal-githook" created="Wed, 1 Apr 2020 21:57:04 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Lingzhi Deng&apos;, &apos;email&apos;: &apos;lingzhi.deng@mongodb.com&apos;, &apos;username&apos;: &apos;ldennis&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-46818&quot; title=&quot;Do untimestamped writes in IdempotencyTest&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-46818&quot;&gt;&lt;del&gt;SERVER-46818&lt;/del&gt;&lt;/a&gt;: Do untimestamped writes in idemptency tests&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/53f344385322f7e9a93779e4a515167e766182cc&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/53f344385322f7e9a93779e4a515167e766182cc&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3019902" author="siyuan.zhou@10gen.com" created="Tue, 31 Mar 2020 19:43:10 +0000"  >&lt;blockquote&gt;&lt;p&gt;Do you think it&apos;s possible that we get enough coverage from the initial sync fuzzer that we might no longer need the idempotency tests?&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;&#160;&lt;br/&gt;
I think we should keep these tests since they are unit tests that give us deterministic test coverage. &lt;/p&gt;

&lt;p&gt;I agree idempotency tests are not testing timestamping behavior and should work with untimestamped writes.&lt;/p&gt;</comment>
                            <comment id="3019256" author="tess.avitabile" created="Tue, 31 Mar 2020 15:29:19 +0000"  >&lt;p&gt;I also support doing untimestamped writes. &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=lingzhi.deng&quot; class=&quot;user-hover&quot; rel=&quot;lingzhi.deng&quot;&gt;lingzhi.deng&lt;/a&gt;, I think I misunderstood the issue when we talked about doing untimestamped writes previously. I think it would be fine even if the test did all untimestamped writes, since I think the main concern of the tests is that the contents of oplog entries and semantics of their application is idempotent.&lt;/p&gt;</comment>
                            <comment id="3019111" author="judah.schvimer" created="Tue, 31 Mar 2020 14:45:58 +0000"  >&lt;p&gt;I don&apos;t think these tests are testing timestamping behavior, just how we apply operations in a linear order, so I support just having them do untimestamped writes.&lt;/p&gt;</comment>
                            <comment id="3019098" author="daniel.gottlieb@10gen.com" created="Tue, 31 Mar 2020 14:41:00 +0000"  >&lt;p&gt;I&apos;m confident untimestamped writes for the &quot;first phase&quot; is logically equivalent to what initial sync can end up with.&lt;/p&gt;</comment>
                            <comment id="3019086" author="lingzhi.deng" created="Tue, 31 Mar 2020 14:37:17 +0000"  >&lt;p&gt;Yes, it shouldnt be too hard to make the tests do untimestamped writes. But that would not match the initial sync behaviors either. I am not sure if that matters / if thats what we want to test.&lt;/p&gt;</comment>
                            <comment id="3019070" author="judah.schvimer" created="Tue, 31 Mar 2020 14:32:17 +0000"  >&lt;p&gt;I think that &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-45827&quot; title=&quot;Expand initial sync fuzzer grammar to include all CRUD document shapes and index DDL ops&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-45827&quot;&gt;SERVER-45827&lt;/a&gt; could make sure that the initial sync fuzzer covers all cases that the idempotency tests cover today. &lt;/p&gt;

&lt;p&gt;Maybe I missed something in the above, but can the test just do all untimestamped writes? I think that would provide the same coverage as today without causing problems in the storage engine. It&apos;s a unittest, so it shouldn&apos;t be too hard to poke through some failpoint to turn off timestamping.&lt;/p&gt;</comment>
                            <comment id="3018952" author="milkie" created="Tue, 31 Mar 2020 13:49:39 +0000"  >&lt;p&gt;The change that made the test pass again was that the WiredTiger code assertions that update timestamps were in order were removed.  It would be better if we could add those assertions back in, as there isn&apos;t really anything else preventing MongoDB code from doing this in the future, where it would most likely be a coding mistake.&lt;/p&gt;</comment>
                            <comment id="3018919" author="tess.avitabile" created="Tue, 31 Mar 2020 13:45:57 +0000"  >&lt;p&gt;What was the change that was made to make the tests pass? Was the invariant turned off just for the test, or was is turned off always? If the invariant was turned off just for the test, then this seems like a fine workaround.&lt;/p&gt;

&lt;p&gt;Another thing we could consider is removing these tests if we now get coverage from other areas. &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=judah.schvimer&quot; class=&quot;user-hover&quot; rel=&quot;judah.schvimer&quot;&gt;judah.schvimer&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=siyuan.zhou&quot; class=&quot;user-hover&quot; rel=&quot;siyuan.zhou&quot;&gt;siyuan.zhou&lt;/a&gt;, do you think it&apos;s possible that we get enough coverage from the initial sync fuzzer that we might no longer need the idempotency tests?&lt;/p&gt;</comment>
                            <comment id="3013286" author="daniel.gottlieb@10gen.com" created="Mon, 30 Mar 2020 17:15:25 +0000"  >&lt;p&gt;A correction, now that I read the spawning &lt;a href=&quot;https://jira.mongodb.org/browse/WT-5799&quot; title=&quot;Dont assume we have ordered timestamps when doing rec append original value&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-5799&quot;&gt;&lt;del&gt;WT-5799&lt;/del&gt;&lt;/a&gt; ticket. WT is not reordering based on timestamp. They had intended to add an invariant that update chains were in timestamp order.&lt;/p&gt;

&lt;p&gt;I don&apos;t think we&apos;ve added any new cases in a production system where MongoDB writes things out of timestamp order. The existing known cases (for a production system) were old-style background index builds (on the index entries themselves) and anything using a ghost timestamp (notably, background index builds completing on a secondary which could write to the catalog entries out of timestamp order).&lt;/p&gt;

&lt;p&gt;The only future project I know of that has the potential for reintroducing out of timestamp order writes would be having clustered indexes and allowing them on secondary (non-&lt;tt&gt;_id&lt;/tt&gt;) indexes. Depending on how things are done, that could bring back the old mmap style &quot;fetch missing documents due to an update oplog entry&quot; behavior for initial sync.&lt;/p&gt;</comment>
                            <comment id="3013171" author="lingzhi.deng" created="Mon, 30 Mar 2020 16:25:06 +0000"  >&lt;p&gt;Yes, ideally we need to somehow turn everything applied into untimestamped (like erase the history/consolidate the update chains) after each round of oplog application so that it better matches the initial sync behaviors in real world. Applying onto different documents would do it too I believe.&lt;/p&gt;

&lt;p&gt;To &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=luke.pearson&quot; class=&quot;user-hover&quot; rel=&quot;luke.pearson&quot;&gt;luke.pearson&lt;/a&gt;, the tests don&apos;t have to work that way. The idea of these tests is to test oplog idempotency even when the changes are already (or partially) reflected in the data. So we cant call &lt;tt&gt;rollback_to_stable&lt;/tt&gt; in these test as that defeats the purposes of these tests. That said, as Dan summarized about, the implementation of the tests that uses oplog application to generate arbitrary versions and then re-apply those entries is not valid and doesnt match real-world mongodb behaviors.&lt;/p&gt;</comment>
                            <comment id="3013140" author="milkie" created="Mon, 30 Mar 2020 16:12:21 +0000"  >&lt;p&gt;Thanks, Dan &amp;#8211; I agree that we should keep this test then.   Perhaps instead of applying the oplog entries to the same document twice, we could do this:&lt;br/&gt;
1. apply oplog entries to {_id: 1}.&lt;br/&gt;
2. read end result of {_id: 1} document and write {_id: 2} document with the same fields, with a timestamp of (0,1) or something.&lt;br/&gt;
3. apply same list oplog entries, but altered to write to {_id: 2} now.  Then check to see if {_id:1} and {_id:2} differ only in _id field.&lt;/p&gt;</comment>
                            <comment id="3012927" author="daniel.gottlieb@10gen.com" created="Mon, 30 Mar 2020 14:58:01 +0000"  >&lt;blockquote&gt;
&lt;p&gt;So if we use testOpsAreIdempotent, shouldn&apos;t there always be a chance where older oplog entries are replayed on top of newer ones?&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;I think the specific problem &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=luke.pearson&quot; class=&quot;user-hover&quot; rel=&quot;luke.pearson&quot;&gt;luke.pearson&lt;/a&gt; is mentioning has to do with WT_MODIFY operations. Typical WT_UPDATES are saving a full copy of the new bson document. A WT_MODIFY just saves the diff. My understanding is that prior to durable history, WiredTiger always pulls updates out of cache overflow in the same order they went in. However after durable history, the update chain is restored in timestamp order. An application writing timestamps out of order could be reordered.&lt;/p&gt;

&lt;p&gt;Reordering a series of WT_MODIFY is likely to result in a bad BSONObject at the end. Given the test likely only cares about the last version after applying all operations, using WT_UPDATES sweeps the problem under the rug (the last update should always be &amp;gt;= any prior update timestamp).&lt;/p&gt;

&lt;blockquote&gt;
&lt;p&gt;I suspect this particular test may simply need to be deleted now&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;The test is still valid in spirit (unless other things exercise idempotency in other ways). Replication still depends on idempotency during initial sync. As &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=lingzhi.deng&quot; class=&quot;user-hover&quot; rel=&quot;lingzhi.deng&quot;&gt;lingzhi.deng&lt;/a&gt; said, the cloned documents are really just a version of a document after applying some arbitrary number of operations. The syncing node can only lower bound (as in timestamp value to start applying from) which oplog entry to start with. It&apos;s just the implementation detail that the test uses oplog application to generate arbitrary versions is no longer valid.&lt;/p&gt;</comment>
                            <comment id="3012924" author="lingzhi.deng" created="Mon, 30 Mar 2020 14:57:00 +0000"  >&lt;p&gt;If my understanding is correct, I am afraid we might need a bigger refactoring (or rewriting) of the idempotency tests in order to match the real-world mongodb behaviors.&lt;/p&gt;</comment>
                            <comment id="3012857" author="lingzhi.deng" created="Mon, 30 Mar 2020 14:38:40 +0000"  >&lt;p&gt;I guess all idempotency unittests in general may have the same problem. It looks like in &lt;tt&gt;testOpsAreIdempotent&lt;/tt&gt;, we always &lt;a href=&quot;https://github.com/mongodb/mongo/blob/a6d3529b264b8b2331faea6a0e645fcf9def8f7f/src/mongo/db/repl/idempotency_test_fixture.cpp#L377&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;apply all oplog entries once first&lt;/a&gt; and then &lt;a href=&quot;https://github.com/mongodb/mongo/blob/a6d3529b264b8b2331faea6a0e645fcf9def8f7f/src/mongo/db/repl/idempotency_test_fixture.cpp#L380-L406&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;re-apply them (or a subset of them) again&lt;/a&gt; based on the &lt;tt&gt;sequenceType&lt;/tt&gt;. So if we use &lt;tt&gt;testOpsAreIdempotent&lt;/tt&gt;, shouldnt there always be a chance where older oplog entries are replayed on top of newer ones?&lt;/p&gt;

&lt;p&gt;I think the general idea for these tests is to simulate re-applying oplog entries while the changes are already reflected in the data &amp;#8211; i.e. idempotency. But in the real-world case of initial sync, I think we always do untimestamped writes during data cloning and so even if the changes are already reflected in the data, they would be untimestamped before re-applying the oplog entries. So it seems that idempotency tests do not quite match with what could happen in the real world. And as Eric pointed out, we shouldnt be replaying older oplog entries on top of newer ones.&lt;/p&gt;</comment>
                            <comment id="2972050" author="milkie" created="Thu, 12 Mar 2020 13:15:54 +0000"  >&lt;p&gt;This test was originally written without timestamps in mind, and I suspect things just sort of worked by accident after we added timestamping to MongoDB.  I suspect this particular test may simply need to be deleted now, as we never replay older oplog entries on top of newer ones without an interim call to recover-to-timestamp.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="399758">SERVER-29944</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>17.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="18953"><![CDATA[v4.4]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Thu, 12 Mar 2020 13:15:54 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        3 years, 44 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            3 years, 44 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>daniel.gottlieb@mongodb.com</customfieldvalue>
            <customfieldvalue>milkie@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>judah.schvimer@mongodb.com</customfieldvalue>
            <customfieldvalue>lingzhi.deng@mongodb.com</customfieldvalue>
            <customfieldvalue>luke.pearson@mongodb.com</customfieldvalue>
            <customfieldvalue>siyuan.zhou@mongodb.com</customfieldvalue>
            <customfieldvalue>tess.avitabile@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hx7b3z:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hwv3iv:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="3768">Repl 2020-04-06</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hx6xdb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>