<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:21:17 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-49949] Reconstructing prepared transactions containing multi-key writes crashes the initial syncing node.</title>
                <link>https://jira.mongodb.org/browse/SERVER-49949</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;&lt;a href=&quot;https://github.com/mongodb/mongo/blob/3e649564b75d7249cd3d970bfa5c00fdaafaf484/src/mongo/db/catalog/index_catalog_entry_impl.cpp#L302&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;On-disk catalog&lt;/a&gt; multi-key update for multi-statement transactions happens in a side transaction block. &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-48010&quot; title=&quot;Substitute ghost timestamp with no-op write in multi-statement txn multikey sidetxn write&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-48010&quot;&gt;&lt;del&gt;SERVER-48010&lt;/del&gt;&lt;/a&gt; replaced ghost timestamp (determined by logical clock) with no-op write for timestamping the on-disk catalog writes on primaries. Since initial sync&#160;&lt;a href=&quot;https://github.com/mongodb/mongo/blob/3e649564b75d7249cd3d970bfa5c00fdaafaf484/src/mongo/db/catalog/index_catalog_entry_impl.cpp#L279&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;doesn&apos;t&lt;/a&gt; set transaction &lt;em&gt;recoveryPrepareOpTime&lt;/em&gt;&#160;while reconstructing prepared transactions, it wrongly calls &lt;a href=&quot;https://github.com/mongodb/mongo/blob/3e649564b75d7249cd3d970bfa5c00fdaafaf484/src/mongo/db/catalog/index_catalog_entry_impl.cpp#L290-L300&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;primary no-op write code path&lt;/a&gt; to acquire timestamp for the catalog write. As a result, we hit &lt;a href=&quot;https://github.com/mongodb/mongo/blob/3e649564b75d7249cd3d970bfa5c00fdaafaf484/src/mongo/db/catalog/index_catalog_entry_impl.cpp#L293&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;this&lt;/a&gt; invariant failure which leads to initial syncing node server crash.&lt;/p&gt;

&lt;p&gt;&lt;b&gt;This happens in both 4.4 and master.&lt;/b&gt;&lt;/p&gt;</description>
                <environment></environment>
        <key id="1423083">SERVER-49949</key>
            <summary>Reconstructing prepared transactions containing multi-key writes crashes the initial syncing node.</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="lingzhi.deng@mongodb.com">Lingzhi Deng</assignee>
                                    <reporter username="suganthi.mani@mongodb.com">Suganthi Mani</reporter>
                        <labels>
                    </labels>
                <created>Tue, 28 Jul 2020 16:15:40 +0000</created>
                <updated>Sun, 29 Oct 2023 22:05:06 +0000</updated>
                            <resolved>Fri, 14 Aug 2020 17:25:39 +0000</resolved>
                                                    <fixVersion>4.4.1</fixVersion>
                    <fixVersion>4.7.0</fixVersion>
                                    <component>Replication</component>
                    <component>Storage</component>
                                        <votes>0</votes>
                                    <watches>9</watches>
                                                                                                                <comments>
                            <comment id="3572565" author="samy.lanka" created="Tue, 19 Jan 2021 22:37:40 +0000"  >&lt;p&gt;The original problem can&apos;t happen on 4.2, but we&apos;re seeing that the prepareTimestamp can be earlier than the oldestTimestamp during startup recovery, which will cause an issue while &lt;a href=&quot;https://github.com/mongodb/mongo/blob/ea77ccc0c2361794e4a5b58c2352031477d717a7/src/mongo/db/catalog/index_catalog_entry_impl.cpp#L315-L331&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;trying to set the timestamp for the catalog multi-key update&lt;/a&gt;. Requesting a backport to 4.2 since the rounding logic will solve this issue as well.&lt;/p&gt;</comment>
                            <comment id="3356064" author="xgen-internal-githook" created="Mon, 24 Aug 2020 18:55:50 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Lingzhi Deng&apos;, &apos;email&apos;: &apos;lingzhi.deng@mongodb.com&apos;, &apos;username&apos;: &apos;ldennis&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-49949&quot; title=&quot;Reconstructing prepared transactions containing multi-key writes crashes the initial syncing node.&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-49949&quot;&gt;&lt;del&gt;SERVER-49949&lt;/del&gt;&lt;/a&gt;: Fix reconstructing prepared transactions containing multi-key writes after initial sync&lt;/p&gt;

&lt;p&gt;(cherry picked from commit 5821e5bc1e2e8c4ed3e791a60a104d57f104caf1)&lt;br/&gt;
Branch: v4.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/4419a5c2a0d099b40b2155dd4def37802286cb7c&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/4419a5c2a0d099b40b2155dd4def37802286cb7c&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3347573" author="suganthi.mani" created="Wed, 19 Aug 2020 17:18:05 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=tess.avitabile&quot; class=&quot;user-hover&quot; rel=&quot;tess.avitabile&quot;&gt;tess.avitabile&lt;/a&gt; 4.2 is not affected by this issue. &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-48010&quot; title=&quot;Substitute ghost timestamp with no-op write in multi-statement txn multikey sidetxn write&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-48010&quot;&gt;&lt;del&gt;SERVER-48010&lt;/del&gt;&lt;/a&gt; which introduced this bug is only available from 4.4 onwards.&lt;/p&gt;</comment>
                            <comment id="3347548" author="tess.avitabile" created="Wed, 19 Aug 2020 17:09:10 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=lingzhi.deng&quot; class=&quot;user-hover&quot; rel=&quot;lingzhi.deng&quot;&gt;lingzhi.deng&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=suganthi.mani&quot; class=&quot;user-hover&quot; rel=&quot;suganthi.mani&quot;&gt;suganthi.mani&lt;/a&gt;, do you know if this affects 4.2?&lt;/p&gt;</comment>
                            <comment id="3339814" author="xgen-internal-githook" created="Fri, 14 Aug 2020 17:24:05 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Lingzhi Deng&apos;, &apos;email&apos;: &apos;lingzhi.deng@mongodb.com&apos;, &apos;username&apos;: &apos;ldennis&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-49949&quot; title=&quot;Reconstructing prepared transactions containing multi-key writes crashes the initial syncing node.&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-49949&quot;&gt;&lt;del&gt;SERVER-49949&lt;/del&gt;&lt;/a&gt;: Fix reconstructing prepared transactions containing multi-key writes after initial sync&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/5821e5bc1e2e8c4ed3e791a60a104d57f104caf1&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/5821e5bc1e2e8c4ed3e791a60a104d57f104caf1&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3334590" author="lingzhi.deng" created="Wed, 12 Aug 2020 14:36:18 +0000"  >&lt;p&gt;After taking a look at the problem, I think this is probably not a one-liner fix because we could be reconstructing a prepared transaction behind the oldest timestamp during initial sync. And if we use the timestamp of the prepare entry to do the multikey write, then we would be writing behind the oldest timestamp. And I think that was why we needed &lt;a href=&quot;https://github.com/mongodb/mongo/blob/354959497599265c78f520a6a80fb378ed6932bd/src/mongo/db/storage/recovery_unit.h#L180-L190&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;&lt;tt&gt;setRoundUpPreparedTimestamps&lt;/tt&gt;&lt;/a&gt;. So I think we will need some kind of roundup as well for the multikey writes. I tried using the max of (prepare timestamp, oldest timestamp) for the multikey write and it seems to work. &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=daniel.gottlieb&quot; class=&quot;user-hover&quot; rel=&quot;daniel.gottlieb&quot;&gt;daniel.gottlieb&lt;/a&gt;, do you think this is a reasonable approach?&lt;/p&gt;

&lt;p&gt;Another idea is to use the &lt;a href=&quot;https://github.com/mongodb/mongo/blob/354959497599265c78f520a6a80fb378ed6932bd/src/mongo/db/repl/initial_syncer.cpp#L544&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;&lt;tt&gt;initialDataTimestamp&lt;/tt&gt;&lt;/a&gt;. I think this is also safe as no reader is allowed at a timestamp earlier than that after the initial sync completes. So it doesnt seem to matter which timestamp to use as long as it is &amp;lt;= &lt;tt&gt;initialDataTimestamp&lt;/tt&gt;.&lt;/p&gt;</comment>
                            <comment id="3318141" author="suganthi.mani" created="Mon, 3 Aug 2020 17:51:46 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=kelsey.schubert&quot; class=&quot;user-hover&quot; rel=&quot;kelsey.schubert&quot;&gt;kelsey.schubert&lt;/a&gt;,  Just want to bring this ticket to your attention as this bug exists on 4.4.  I think as per &quot;4.4 Backports Post GA schedule by team&quot;, for replication team, it&apos;s scheduled as August 21. Do you see any urgency for this ticket to be backported before August 21?&lt;/p&gt;</comment>
                            <comment id="3311300" author="suganthi.mani" created="Wed, 29 Jul 2020 07:15:20 +0000"  >&lt;p&gt;We don&apos;t hit this problem for startup/rollback recovery because we &lt;a href=&quot;https://github.com/mongodb/mongo/blob/21a3ee6079eb3cc83a5210cf53d98840cbc83a20/src/mongo/db/repl/transaction_oplog_application.cpp#L436-L437&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;set&lt;/a&gt; &lt;em&gt;recoveryPrepareOpTime&lt;/em&gt; to prepareTimestamp when the node&apos;s oplog application mode is recovering. As a result, this prepareTimestamp will be used to &lt;a href=&quot;https://github.com/mongodb/mongo/blob/3e649564b75d7249cd3d970bfa5c00fdaafaf484/src/mongo/db/catalog/index_catalog_entry_impl.cpp#L282&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;timestamp&lt;/a&gt; catalog multi-key update. We should have a similar solution to tackle the initial sync scenario case. It&apos;s actually safe to use prepare timestamp for catalog multi-key update during initial sync &amp;amp; startup/rollback recovery because that timestamp is guaranteed to be less than or equal to the commit timestamp of the transaction and the contract is that multi-key write must occur at a time &amp;lt;= the first write that makes an index multi-key (See &lt;a href=&quot;https://jira.mongodb.org/secure/EditComment!default.jspa?id=537293&amp;amp;commentId=1879676&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;this&lt;/a&gt; comment).&lt;/p&gt;

&lt;p&gt;Note: &lt;a href=&quot;https://github.com/mongodb/mongo/blob/3e649564b75d7249cd3d970bfa5c00fdaafaf484/src/mongo/db/catalog/index_catalog_entry_impl.cpp#L246&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;Secondary&lt;/a&gt; oplog application doesn&apos;t use the side transaction block for catalog multi-key update because the opCtx that performs catalog write is &lt;a href=&quot;https://github.com/mongodb/mongo/blob/3e649564b75d7249cd3d970bfa5c00fdaafaf484/src/mongo/db/repl/oplog_applier_impl.cpp#L761&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;different&lt;/a&gt; from the opCtx which apply prepare transaction.&lt;/p&gt;

&lt;p&gt;Fix should be one-liner change.&#160; &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=tess.avitabile&quot; class=&quot;user-hover&quot; rel=&quot;tess.avitabile&quot;&gt;tess.avitabile&lt;/a&gt;&#160;Do you know what&apos;s the priority of this ticket would be? We consistently crash the initial syncing node when reconstructing prepared transactions containing multi-key writes on 4.4.&lt;/p&gt;

&lt;p&gt;CC &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=daniel.gottlieb&quot; class=&quot;user-hover&quot; rel=&quot;daniel.gottlieb&quot;&gt;daniel.gottlieb&lt;/a&gt;&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="859350">SERVER-42251</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1343348">SERVER-48010</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="1595830">SERVER-53932</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>8.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="18953"><![CDATA[v4.4]]></customfieldvalue>
    <customfieldvalue key="16775"><![CDATA[v4.2]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 12 Aug 2020 14:36:18 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        3 years, 3 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            3 years, 3 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_16465" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Linked BF Score</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>20.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>lingzhi.deng@mongodb.com</customfieldvalue>
            <customfieldvalue>samy.lanka@mongodb.com</customfieldvalue>
            <customfieldvalue>suganthi.mani@mongodb.com</customfieldvalue>
            <customfieldvalue>tess.avitabile@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hxx50n:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hxjy7b:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="4140">Repl 2020-08-24</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hxwr9z:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>