<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:54:14 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-40176] Cursor seekExact should not use WT_CURSOR:search_near to avoid unintentional prepare conflicts</title>
                <link>https://jira.mongodb.org/browse/SERVER-40176</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;As described in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-40167&quot; title=&quot;Index key removal should not encounter prepare conflicts on unrelated keys&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-40167&quot;&gt;&lt;del&gt;SERVER-40167&lt;/del&gt;&lt;/a&gt;, &lt;a href=&quot;https://github.com/mongodb/mongo/blob/9acc250dc9b6ce92f1cee13b404a408e17804564/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp#L1029&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;seeking using search_near&lt;/a&gt; can cause operations to encounter unintended prepare conflicts when the queried key is not found. &lt;/p&gt;</description>
                <environment></environment>
        <key id="717608">SERVER-40176</key>
            <summary>Cursor seekExact should not use WT_CURSOR:search_near to avoid unintentional prepare conflicts</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="2">Won&apos;t Fix</resolution>
                                        <assignee username="louis.williams@mongodb.com">Louis Williams</assignee>
                                    <reporter username="louis.williams@mongodb.com">Louis Williams</reporter>
                        <labels>
                            <label>txn_storage</label>
                    </labels>
                <created>Fri, 15 Mar 2019 20:50:05 +0000</created>
                <updated>Thu, 26 Oct 2023 14:21:22 +0000</updated>
                            <resolved>Mon, 17 Jun 2019 16:23:57 +0000</resolved>
                                                                    <component>Replication</component>
                    <component>Storage</component>
                                        <votes>0</votes>
                                    <watches>15</watches>
                                                                                                                <comments>
                            <comment id="2237687" author="louis.williams" created="Tue, 7 May 2019 20:08:56 +0000"  >&lt;p&gt;We decided we will not be making any changes to the storage API, and instead perform inserts as inserts, instead of upserts, during steady-state replication for &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-40177&quot; title=&quot;Enforce prepare conflicts on secondaries&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-40177&quot;&gt;&lt;del&gt;SERVER-40177&lt;/del&gt;&lt;/a&gt;.&lt;/p&gt;</comment>
                            <comment id="2230943" author="judah.schvimer" created="Wed, 1 May 2019 19:04:48 +0000"  >&lt;p&gt;Removing from the prepare epic so this can be prioritized by the storage team separately from the rest of the project.&lt;/p&gt;</comment>
                            <comment id="2228634" author="alexander.gorrod" created="Tue, 30 Apr 2019 04:19:35 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=louis.williams&quot; class=&quot;user-hover&quot; rel=&quot;louis.williams&quot;&gt;louis.williams&lt;/a&gt; I&apos;ve booked a meeting to talk about this work.&lt;/p&gt;</comment>
                            <comment id="2223484" author="louis.williams" created="Wed, 24 Apr 2019 16:13:28 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=michael.cahill&quot; class=&quot;user-hover&quot; rel=&quot;michael.cahill&quot;&gt;michael.cahill&lt;/a&gt; thanks for the summary. This is what I understand are the changes required. I don&apos;t think we&apos;ll be able to avoid unintentional conflicts on non-unique indexes for any case, but I&apos;m not sure of a solution where that&apos;s avoidable.&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-40167&quot; title=&quot;Index key removal should not encounter prepare conflicts on unrelated keys&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-40167&quot;&gt;&lt;del&gt;SERVER-40167&lt;/del&gt;&lt;/a&gt;: cursor restore after remove
	&lt;ul&gt;
		&lt;li&gt;&lt;b&gt;Work required&lt;/b&gt;: return the positioned key when &lt;tt&gt;searh_near&lt;/tt&gt; hits a prepare conflict. If the key returned matches the key requested, then the caller will observe the prepare conflict. If the returned key doesn&apos;t not match, allow the caller to use next() to potentially hit any adjacent records that may be prepared. This is prone to prepare conflicts for keys on non-unique indexes.&#160;&lt;/li&gt;
	&lt;/ul&gt;
	&lt;/li&gt;
	&lt;li&gt;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-40176&quot; title=&quot;Cursor seekExact should not use WT_CURSOR:search_near to avoid unintentional prepare conflicts&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-40176&quot;&gt;&lt;del&gt;SERVER-40176&lt;/del&gt;&lt;/a&gt; (this ticket)
	&lt;ul&gt;
		&lt;li&gt;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-40177&quot; title=&quot;Enforce prepare conflicts on secondaries&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-40177&quot;&gt;&lt;del&gt;SERVER-40177&lt;/del&gt;&lt;/a&gt; is just a symptom of this issue, because updates on secondaries use seekExact via upserts.&lt;/li&gt;
		&lt;li&gt;&lt;b&gt;Work required&lt;/b&gt;: use &lt;tt&gt;search_prefix&lt;/tt&gt; to only return records matching a prefix. If the first match is prepared, then the prepare conflict is observed.&#160;If there is no match, WT_NOTFOUND is treated as EOF by MongoDB, which is fine because seekExact does not expect to get keys that do not match. This is also prone to hit prepare conflicts for non-unique indexes.&#160;&lt;/li&gt;
	&lt;/ul&gt;
	&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;If this sounds right and appropriate, I can open WT tickets for both changes.&lt;/p&gt;</comment>
                            <comment id="2212949" author="michael.cahill" created="Mon, 15 Apr 2019 06:54:52 +0000"  >&lt;p&gt;Here is my attempt to summarize my understanding of the various issues and proposed WT API changes:&lt;/p&gt;
&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&#160;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-40167&quot; title=&quot;Index key removal should not encounter prepare conflicts on unrelated keys&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-40167&quot;&gt;&lt;del&gt;SERVER-40167&lt;/del&gt;&lt;/a&gt;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-40176&quot; title=&quot;Cursor seekExact should not use WT_CURSOR:search_near to avoid unintentional prepare conflicts&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-40176&quot;&gt;&lt;del&gt;SERVER-40176&lt;/del&gt;&lt;/a&gt;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-40177&quot; title=&quot;Enforce prepare conflicts on secondaries&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-40177&quot;&gt;&lt;del&gt;SERVER-40177&lt;/del&gt;&lt;/a&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&#160;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;cursor restore after remove&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;seekExact uses search_near, can get spurious prepare conflicts&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;prepare conflicts in secondary oplog application&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;ignore_prepare&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;maybe okay but we&apos;re trying to get away from ignoring prepared updates&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;not clear it is always safe to ignore conflicts, definitely safe to ignore all conflicts during update transactions&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;should be safe but ignoring conflicts could mask bugs&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;search prefix&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;not clear this helps?&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;finds the first match: what if it is prepared? if there is no match, returns WT_NOTFOUND, how is that distinguished from EOF?&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;should fix this: secondaries only search _id index&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;get key after prepare conflict&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;could make the restore successful and only block if cursor-&amp;gt;next is called&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;could check the key to decide if the prepare conflict is genuine. Then what &amp;#8211; e.g., if the returned key is less than the search key, need to call cursor-&amp;gt;next&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;as for &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-40176&quot; title=&quot;Cursor seekExact should not use WT_CURSOR:search_near to avoid unintentional prepare conflicts&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-40176&quot;&gt;&lt;del&gt;SERVER-40176&lt;/del&gt;&lt;/a&gt;&lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;


&lt;p&gt;I&apos;m concerned that &lt;tt&gt;search_prefix&lt;/tt&gt; doesn&apos;t help in all cases.&#160; In particular, if documents share an ordinary, non-unique index key, then a prepared transaction in an unrelated document that happens to share the same index key value could still cause blocking when accessing an adjacent document.&lt;/p&gt;

&lt;p&gt;If we can agree on a complete set of changes that mean prepare conflicts only cause blocking when accessing the documents involved in a prepared transaction, then we can map out and schedule the WT changes.&lt;/p&gt;

&lt;p&gt;If we&apos;re saying that prepared conflicts with logically unrelated documents are unavoidable in some cases, maybe the &lt;tt&gt;search_prefix&lt;/tt&gt; approach is sufficient, but then I&apos;d like clearer answers to the questions above.&lt;/p&gt;</comment>
                            <comment id="2199256" author="louis.williams" created="Tue, 2 Apr 2019 15:55:13 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=michael.cahill&quot; class=&quot;user-hover&quot; rel=&quot;michael.cahill&quot;&gt;michael.cahill&lt;/a&gt; thanks for talking through this.&lt;/p&gt;
&lt;blockquote&gt;&lt;p&gt;would it be sufficient to simply ignore all updates that are part of prepared transactions and allow writes? Or are there cases where the conflict is real and needs to be resolved to provide correct semantics.&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;I don&apos;t think in any case it would be 100% safe to allow writes when ignoring updates that are part of prepared transactions. To be honest, this is hard to reason about. In the case of key removal (&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-40167&quot; title=&quot;Index key removal should not encounter prepare conflicts on unrelated keys&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-40167&quot;&gt;&lt;del&gt;SERVER-40167&lt;/del&gt;&lt;/a&gt;), the key selected for removal could be part of a prepared transaction, and that should encounter a legitimate prepare conflict. It would be incorrect to ignore that.&lt;/p&gt;
&lt;blockquote&gt;&lt;p&gt;what would the proposed search_prefix method do if there were entries that match the prefix, but some of them were part of prepared transactions?&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;In this case, MongoDB would have to traverse through a small set of keys, but &lt;tt&gt;search_prefix&lt;/tt&gt; should still encounter a WT_PREPARE_CONFLICT. The motivation behind this change is to not allow cursors to return records that it wouldn&apos;t find useful, especially if that record is prepared. As long as the only keys a cursor iterates over match the provided prefix, this seems correct to me.&lt;/p&gt;
&lt;blockquote&gt;&lt;p&gt;Another case we would have to think about is what if there are matching entries that were removed as part of a prepared transaction (these would be visible with ignore_prepared=true)&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;In this case, again, I think it&apos;s fine to return a WT_PREPARE_CONFLICT. Non-unique indexes can return multiple documents for a single key, and if that requires traversing a small set, then it should not skip prepared updates. I also think this case would effectively maintain the current behavior of &lt;tt&gt;search_near&lt;/tt&gt;.&lt;/p&gt;
&lt;blockquote&gt;&lt;p&gt;Again imagine the case where there are multiple entries that match the prefix and the first one happens to be prepared: that could return WT_PREPARE_CONFLICT and set the key in the cursor, allowing MongoDB to check that the entry matches. But then what &#8211; do we need to be able to move to the next record? What if that is prepared? etc.&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;This raises a good question about how MongoDB uses cursors, especially for document removal. Even if cursor restoration used &lt;tt&gt;search_prefix&lt;/tt&gt; after deleting a key, I would expect the return value to be WT_NOTFOUND. Then what? How would MongoDB differentiate an EOF from a repositioned cursor that landed on a recently deleted key? Maybe it&apos;s possible that if we decide to return the key after hitting a WT_PREPARE_CONFLICT, the &lt;tt&gt;_endPosition&lt;/tt&gt; from the Index scan can be used to consider whether or not the scan is at EOF. Also, I don&apos;t imagine this being an issue for the upsert case on secondaries.&lt;/p&gt;</comment>
                            <comment id="2183896" author="louis.williams" created="Mon, 18 Mar 2019 15:45:58 +0000"  >&lt;p&gt;Quoted from &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-40167&quot; title=&quot;Index key removal should not encounter prepare conflicts on unrelated keys&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-40167&quot;&gt;&lt;del&gt;SERVER-40167&lt;/del&gt;&lt;/a&gt;&lt;/p&gt;
&lt;blockquote&gt;&lt;p&gt;I think the reason&#160; &lt;span class=&quot;error&quot;&gt;&amp;#91;why this will not block testing&amp;#93;&lt;/span&gt;&#160;is simply that each passthrough test operates on its own session and its own collection. We&#8217;ll never see this issue there because it only happens when different sessions are operating on the same collection. Even for FSM tests, prepared transactions always end up getting committed or aborted. So even if an operation encounters a prepare conflict, the conflicting prepared transactions will eventually finish, and the conflict will resolve. This issue seems isolated only to targeted replica set tests where prepared transactions are held open and another operation is attempted.&lt;/p&gt;&lt;/blockquote&gt;</comment>
                            <comment id="2183814" author="judah.schvimer" created="Mon, 18 Mar 2019 15:00:08 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=louis.williams&quot; class=&quot;user-hover&quot; rel=&quot;louis.williams&quot;&gt;louis.williams&lt;/a&gt;, IIUC, there will be BFs but we cannot predict how many? If so I lean towards marking as a blocker to escalate the priority.&lt;/p&gt;</comment>
                            <comment id="2183785" author="louis.williams" created="Mon, 18 Mar 2019 14:45:23 +0000"  >&lt;p&gt;I think it is important to do this for correctness because it would be especially difficult to diagnose from a user perspective. I&apos;m not sure if this is blocking passthrough testing, however, because it&apos;s hard to predict the magnitude of failures that might occur.&lt;/p&gt;</comment>
                            <comment id="2182424" author="judah.schvimer" created="Sat, 16 Mar 2019 22:11:30 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=louis.williams&quot; class=&quot;user-hover&quot; rel=&quot;louis.williams&quot;&gt;louis.williams&lt;/a&gt;, am I correct that this is effectively blocking a lot of our passthrough testing for sharded transactions because we would get hard to diagnose BFs as a result? If so, I will mark this as a &quot;Blocker&quot; to escalate its priority. &lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                                                <inwardlinks description="is depended on by">
                                        <issuelink>
            <issuekey id="717611">SERVER-40177</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="688879">WT-4580</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="727195">SERVER-40421</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="2483579">SERVER-82457</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="717511">SERVER-40167</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>10.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>3.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Sat, 16 Mar 2019 22:11:30 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        4 years, 40 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>randolph@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            4 years, 40 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_16465" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Linked BF Score</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>5.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>alexander.gorrod@mongodb.com</customfieldvalue>
            <customfieldvalue>judah.schvimer@mongodb.com</customfieldvalue>
            <customfieldvalue>louis.williams@mongodb.com</customfieldvalue>
            <customfieldvalue>michael.cahill@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|huqxyf:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr78tb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="2829">Storage NYC 2019-04-08</customfieldvalue>
    <customfieldvalue id="2908">Storage NYC 2019-05-06</customfieldvalue>
    <customfieldvalue id="2909">Storage NYC 2019-05-20</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|huqk7r:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>