<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 06:18:13 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-71156] Fatal Assertion 40507 UnrecoverableRollbackError and Mongo Crash with 4.0.27 WT</title>
                <link>https://jira.mongodb.org/browse/SERVER-71156</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;&lt;b&gt;Problem Description&lt;/b&gt;&lt;/p&gt;

&lt;p&gt;With mongo 4.0 WT StorageEngine, frequently experiencing UnrecoverableRollbackError Crash.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&lt;b&gt;Setup Details&lt;/b&gt;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Mongo Server Version: v4.0.27&lt;/p&gt;

&lt;p&gt;git version: d47b151b55f286546e7c7c98888ae0577856ca20&lt;/p&gt;

&lt;p&gt;OpenSSL version: OpenSSL 1.1.1k&#160; FIPS 25 Mar 2021&lt;/p&gt;

&lt;p&gt;distmod: rhel80&lt;/p&gt;

&lt;p&gt;Mongod Options:&#160; options: { net: &lt;/p&gt;
{ bindIpAll: true, ipv6: true, port: 27717 }
&lt;p&gt;, operationProfiling: { slowOpThresholdMs: 500 }, processManagement: { fo7.pid&quot; }, replication: { &lt;b&gt;enableMajorityReadConcern: false&lt;/b&gt;, oplogSizeMB: &lt;b&gt;5120&lt;/b&gt;, replSet: &quot;set01f&quot; }, security: { keyFile: &quot;/root/.dbkey&quot; }, storage: &lt;/p&gt;
{ dbPath: &quot;/var/data/sessions.1/f&quot;, *enginecheSizeGB: 22.0* }
&lt;p&gt; } }, systemLog: { destination: &quot;file&quot;, logAppend: true, path: &quot;/var/log/mongodb-27717.log&quot;, quiet: true } }&lt;/p&gt;

&lt;p&gt;ReplicaSet with 4 data bearing members and 1 arbiter&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&lt;b&gt;RepSet Settings&lt;/b&gt;&lt;/p&gt;

&lt;p&gt;&quot;protocolVersion&quot; : NumberLong(1),&lt;/p&gt;

&lt;p&gt;&quot;writeConcernMajorityJournalDefault&quot; : false,&lt;/p&gt;

&lt;p&gt;&quot;settings&quot; : {&lt;/p&gt;

&lt;p&gt;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160; &quot;chainingAllowed&quot; : true,&lt;/p&gt;

&lt;p&gt;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160; &quot;heartbeatIntervalMillis&quot; : 2000,&lt;/p&gt;

&lt;p&gt;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160; &quot;heartbeatTimeoutSecs&quot; : 1,&lt;/p&gt;

&lt;p&gt;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160; &quot;electionTimeoutMillis&quot; : 10000,&lt;/p&gt;

&lt;p&gt;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160; &quot;catchUpTimeoutMillis&quot; : -1,&lt;/p&gt;

&lt;p&gt;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160; &quot;catchUpTakeoverDelayMillis&quot; : 30000,&lt;/p&gt;

&lt;p&gt;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160; &quot;getLastErrorModes&quot; : &lt;/p&gt;
{

&#160;

&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160; &#160;&#160;&#160;}
&lt;p&gt;,&lt;/p&gt;

&lt;p&gt;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160; &quot;getLastErrorDefaults&quot; : &lt;/p&gt;
{

&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160; &quot;w&quot; : 1,

&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160; &quot;wtimeout&quot; : 0

&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160;&#160; }
&lt;p&gt;,&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&lt;b&gt;Test Case while the crash reported&lt;/b&gt;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;We have migrated from 3.6.17 mmapV1 storage Engine to Mongo 4.0.27 WT Storage Engine recently. The 4.0.27 Upgrade is required inorder for us to migrate to 4.2 Version. This is the First time we have been running our application with WT storage Engine. We configured WT Cache size of 22GB and oplog of size 5GB(have been using this opLog for more than 7 years).&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;The test case runs a delete of records at 700 TPS per replicaSet. While this test is running, we see few replicaSet are going to RECOVERING State. When we analyzed the log, we see&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;replication-3928&amp;#93;&lt;/span&gt; Restarting oplog query due to error: CappedPositionLost: error in fetcher batch callback :: caused by :: CollectionScan died due tast seen record id: RecordId(7161504155264090918). Last fetched optime (with hash): { ts: Timestamp(1667417622, 805), t: 104 }&lt;span class=&quot;error&quot;&gt;&amp;#91;5387250820607207456&amp;#93;&lt;/span&gt;. Restarts remaining: 1&lt;/p&gt;

&lt;p&gt;2022-11-02T19:48:13.531+0000 I REPL&#160;&#160;&#160;&#160; &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; Starting rollback due to OplogStartMissing: Our last op time fetched: { ts: Timestamp(1667417622, 805), t: 104 }. source&apos;s GTE: &lt;/p&gt;
{&#160; (5387250820607207456/5740888219355370057)

&#160;

2022-11-02T19:49:31.633+0000 F ROLLBACK [rsBackgroundSync] Unable to complete rollback. A full resync may be needed: UnrecoverableRollbackError: need to rollback, but unable to determine cochingDocument: reached beginning of remote oplog: \{them: nd2bwa3psm11vb:27717, theirTime: Timestamp(1667417690, 868)}

&lt;p&gt;2022-11-02T19:49:31.633+0000 F -&#160;&#160;&#160;&#160;&#160;&#160;&#160; &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; Fatal Assertion 40507 at src/mongo/db/repl/rs_rollback.cpp 1567&lt;/p&gt;

&lt;p&gt;2022-11-02T19:49:31.634+0000 F -&#160;&#160;&#160;&#160;&#160;&#160;&#160; &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; \n\n***aborting after fassert() failure\n\n&lt;/p&gt;

&lt;p&gt;2022-11-02T19:49:34.853+0000 I CONTROL&#160; &lt;span class=&quot;error&quot;&gt;&amp;#91;main&amp;#93;&lt;/span&gt; ***** SERVER RESTARTED *****&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;When the server restarts, its recovering from a unstable checkpoint and fail to syncup with any members stating they are too stale and finally going to maintenance mode.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;2022-11-02T19:49:41.746+0000 I REPL&#160;&#160;&#160;&#160; &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Recovering from an unstable checkpoint (top of oplog: { ts: Timestamp(1667417622, 805), t: 104 }, appliedThrough: &lt;/p&gt;
{ ts: Timestamp(166

2022-11-02T19:49:41.746+0000 I REPL&#160;&#160;&#160;&#160; [initandlisten] Starting recovery oplog application at the appliedThrough: \{ ts: Timestamp(1667417622, 805), t: 104 }
&lt;p&gt;, through the top of the oplog:&lt;/p&gt;

&lt;p&gt;2022-11-02T19:49:41.746+0000 I REPL&#160;&#160;&#160;&#160; &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; No oplog entries to apply for recovery. Start point is at the top of the oplog.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Server logs and diagnostics.data are attached. Nd2bwa3psm12vb, nd2bwa3psm11va are the affected replicaSets. But attached all the 4 replicaMembers logs for reference.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;</description>
                <environment></environment>
        <key id="2177143">SERVER-71156</key>
            <summary>Fatal Assertion 40507 UnrecoverableRollbackError and Mongo Crash with 4.0.27 WT</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="yuan.fang@mongodb.com">Yuan Fang</assignee>
                                    <reporter username="veramasu@hcl.com">venkataramans rama</reporter>
                        <labels>
                    </labels>
                <created>Mon, 7 Nov 2022 23:47:57 +0000</created>
                <updated>Tue, 15 Nov 2022 17:15:10 +0000</updated>
                            <resolved>Tue, 15 Nov 2022 17:15:10 +0000</resolved>
                                                                                        <votes>1</votes>
                                    <watches>5</watches>
                                                                                                                <comments>
                            <comment id="4980290" author="JIRAUSER1270794" created="Mon, 14 Nov 2022 20:46:30 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=veramasu%40hcl.com&quot; class=&quot;user-hover&quot; rel=&quot;veramasu@hcl.com&quot;&gt;veramasu@hcl.com&lt;/a&gt; ,&lt;/p&gt;

&lt;p&gt;Thank you for your report. Looking at the mongod.log, it appears that the secondary node started &quot;&lt;a href=&quot;https://www.mongodb.com/docs/manual/core/replica-set-rollbacks/#rollbacks-during-replica-set-failover&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;rollback&lt;/a&gt; due to OplogStartMissing&quot;, then the error below suggests the node fell behind the oplog window and the rollback failed because the common point could not be determined:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2022-11-02T19:40:10.769+0000 F ROLLBACK [rsBackgroundSync] Unable to complete rollback. A full resync may be needed: UnrecoverableRollbackError: need to rollback, but unable to determine common point between local and remote oplog: NoMatchingDocument: reached beginning of remote oplog: {them: nd2bwa3psm11va:27717, theirTime: Timestamp(1667417174, 675)}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;Additionally, the message below also indicates that node &lt;tt&gt;nd2bwa3psm11vb:27717&lt;/tt&gt; has fallen off the oplog:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2022-11-02T19:40:29.986+0000 I REPL [replication-0] We are too stale to use nd2bwa3psm11va:27717 as a sync source. Blacklisting this sync source because our last fetched timestamp: Timestamp(1667417084, 143) is before their earliest timestamp: Timestamp(1667417202, 127) for 1min until: 2022-11-02T19:41:29.986+0000&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;A replica set member becomes &#8220;stale&#8221; when its replication process falls so far behind that the primary overwrites oplog entries the member has not yet replicated. You can find more information &lt;a href=&quot;https://www.mongodb.com/docs/atlas/reference/alert-resolutions/replication-oplog/#monitor-your-progress&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;here&lt;/a&gt;. You may also want to check this &lt;a href=&quot;https://www.mongodb.com/community/forums/t/alert-replication-oplog-window-has-gone-below-1-hours/114043/2&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;comment&lt;/a&gt; in a thread that discusses &quot;mitigating the risk of falling off the oplog&quot;.&lt;/p&gt;

&lt;p&gt;With all being said, to remediate it, you will need to completely resynchronize the member by performing an initial sync. Also, you may consider increasing the size of the oplog on both members, if the incident reoccurs after upgrading to v4.2 (v4.0 is no longer supported).&lt;/p&gt;

&lt;p&gt;For issues you may encounter, we&apos;d like to encourage you to start by asking our community for help by posting on the &lt;a href=&quot;https://www.mongodb.com/community/forums&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;MongoDB Developer Community Forums&lt;/a&gt;. If the discussion there leads you to suspect a bug in the MongoDB server, then we&apos;d want to investigate it as a possible bug here in the SERVER project.&lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
Yuan&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="356068">SERVER-28068</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="511278">SERVER-33878</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="275717">SERVER-23392</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="413637" name="Mongo-logs_11_8_2022.tar" size="113435136" author="veramasu@hcl.com" created="Mon, 7 Nov 2022 23:46:18 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 14 Nov 2022 20:46:30 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        1 year, 12 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>yuan.fang@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            1 year, 12 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>veramasu@hcl.com</customfieldvalue>
            <customfieldvalue>yuan.fang@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i1h5l3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|i0zs0w:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[yuan.fang@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i1grqf:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>