<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:59:09 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-22016] Fatal assertion 28723 trying to rollback applyOps on a CSRS config server</title>
                <link>https://jira.mongodb.org/browse/SERVER-22016</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;We deploy replica set between multi location(data center), and We use from mongodb 3.2.&lt;br/&gt;
Here is some problems on some machines, The configsvrs after some machines after some minutes down.&lt;br/&gt;
We create replica set for configsvr from 12 members, that The replica set have 7 vote for primary and the rest is vote=0 and priority=0.&lt;br/&gt;
How to fix my issue?&lt;/p&gt;

&lt;p&gt;configsvr logs:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-26T23:46:39.737+0330 I REPL     [ReplicationExecutor] Error in heartbeat request to shard2:47041; HostUnreachable Connection refused&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-26T23:46:40.425+0330 F REPL     [rsBackgroundSync] can&apos;t rollback this command yet: { applyOps: [ { op: &quot;u&quot;, b: true, ns: &quot;config.chunks&quot;, o: { _id: &quot;INN.destinations-destination_code_MinKey&quot;, lastmod: Timestamp 1000|1, lastmodEpoch: ObjectId(&apos;567eb3043d88f022d5c0cf86&apos;), ns: &quot;INN.destinations&quot;, min: { destination_code: MinKey }, max: { destination_code: 0.0 }, shard: &quot;shard1_1&quot; }, o2: { _id: &quot;INN.destinations-destination_code_MinKey&quot; } }, { op: &quot;u&quot;, b: true, ns: &quot;config.chunks&quot;, o: { _id: &quot;INN.destinations-destination_code_0.0&quot;, lastmod: Timestamp 1000|2, lastmodEpoch: ObjectId(&apos;567eb3043d88f022d5c0cf86&apos;), ns: &quot;INN.destinations&quot;, min: { destination_code: 0.0 }, max: { destination_code: MaxKey }, shard: &quot;shard1_1&quot; }, o2: { _id: &quot;INN.destinations-destination_code_0.0&quot; } } ], maxTimeMS: 30000 }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-26T23:46:40.425+0330 I REPL     [rsBackgroundSync] cmdname=applyOps&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-26T23:46:40.425+0330 E REPL     [rsBackgroundSync] replica set fatal exception&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-26T23:46:40.425+0330 I REPL     [rsBackgroundSync] rollback finished&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-26T23:46:40.425+0330 I -        [rsBackgroundSync] Fatal assertion 28723 UnrecoverableRollbackError need to rollback, but unable to determine common point between local and remote oplog: replica set fatal exception @ 18752&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-26T23:46:40.425+0330 I -        [rsBackgroundSync]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;***aborting after fassert() failure&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</description>
                <environment></environment>
        <key id="245099">SERVER-22016</key>
            <summary>Fatal assertion 28723 trying to rollback applyOps on a CSRS config server</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="schwerin@mongodb.com">Andy Schwerin</assignee>
                                    <reporter username="Ali.Hallaji">Ali Hallaji</reporter>
                        <labels>
                            <label>code-and-test</label>
                    </labels>
                <created>Sun, 27 Dec 2015 08:23:26 +0000</created>
                <updated>Thu, 6 Apr 2023 15:43:32 +0000</updated>
                            <resolved>Mon, 4 Jan 2016 21:06:17 +0000</resolved>
                                                    <fixVersion>3.2.3</fixVersion>
                    <fixVersion>3.3.0</fixVersion>
                                    <component>Sharding</component>
                                        <votes>0</votes>
                                    <watches>9</watches>
                                                                                                                <comments>
                            <comment id="1158296" author="xgen-internal-githook" created="Fri, 29 Jan 2016 19:47:40 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;monkey101&apos;, u&apos;name&apos;: u&apos;Dan Pasette&apos;, u&apos;email&apos;: u&apos;dan@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-22016&quot; title=&quot;Fatal assertion 28723 trying to rollback applyOps on a CSRS config server&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-22016&quot;&gt;&lt;del&gt;SERVER-22016&lt;/del&gt;&lt;/a&gt; fix unit test for 3.2 backport&lt;br/&gt;
Branch: v3.2&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/0f10a675edbb6aa9c823bb8bf9f9b20027fea01f&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/0f10a675edbb6aa9c823bb8bf9f9b20027fea01f&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1157992" author="xgen-internal-githook" created="Fri, 29 Jan 2016 17:09:58 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;andy10gen&apos;, u&apos;name&apos;: u&apos;Andy Schwerin&apos;, u&apos;email&apos;: u&apos;schwerin@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-22016&quot; title=&quot;Fatal assertion 28723 trying to rollback applyOps on a CSRS config server&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-22016&quot;&gt;&lt;del&gt;SERVER-22016&lt;/del&gt;&lt;/a&gt; Support rolling back certain applyOps oplog entries in replication.&lt;/p&gt;

&lt;p&gt;Specifically, support rolling back insert, update and delete applyOps operations&lt;br/&gt;
such as those generated by sharding&apos;s moveChunk, splitChunk and mergeChunks&lt;br/&gt;
operations.&lt;/p&gt;

&lt;p&gt;(cherry picked from commit fc81fdee1da1d949f80075c8a88998fa3b0c5e78)&lt;br/&gt;
Branch: v3.2&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/5ca8d6e28255e9519db61d5d25807d70a3000ba7&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/5ca8d6e28255e9519db61d5d25807d70a3000ba7&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1147749" author="ali.hallaji" created="Tue, 19 Jan 2016 19:49:30 +0000"  >&lt;p&gt;Thank you all friends for your comments.&lt;/p&gt;</comment>
                            <comment id="1147694" author="dan@10gen.com" created="Tue, 19 Jan 2016 19:27:08 +0000"  >&lt;p&gt;Yes, you can and should upgrade to 3.2.1.  The problem fixed in this ticket is very specific and does have a workaround/fix if you were to hit it again.  The upgrade from 3.2.1 to 3.2.2 is an upgrade to the executable only and should not be a problem.&lt;/p&gt;</comment>
                            <comment id="1147494" author="ali.hallaji" created="Tue, 19 Jan 2016 17:31:49 +0000"  >&lt;p&gt;Can I install mongodb 3.2.1 for production and wait for new version(3.2.2).&lt;br/&gt;
Can mongodb 3.2.1 upgrade to 3.2.2 without crash and data lost or any problem when new version will be release?&lt;/p&gt;</comment>
                            <comment id="1146077" author="dan@10gen.com" created="Sun, 17 Jan 2016 16:06:14 +0000"  >&lt;p&gt;3.2.2 won&apos;t be ready until mid- next month&lt;/p&gt;</comment>
                            <comment id="1145997" author="ali.hallaji" created="Sun, 17 Jan 2016 06:00:03 +0000"  >&lt;p&gt;Hi Andy,&lt;br/&gt;
When version 3.2.2 is released?&lt;br/&gt;
We&apos;ll wait for new version, But how much long to release that version?&lt;/p&gt;</comment>
                            <comment id="1125687" author="xgen-internal-githook" created="Mon, 4 Jan 2016 21:05:52 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;andy10gen&apos;, u&apos;name&apos;: u&apos;Andy Schwerin&apos;, u&apos;email&apos;: u&apos;schwerin@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-22016&quot; title=&quot;Fatal assertion 28723 trying to rollback applyOps on a CSRS config server&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-22016&quot;&gt;&lt;del&gt;SERVER-22016&lt;/del&gt;&lt;/a&gt; Support rolling back certain applyOps oplog entries in replication.&lt;/p&gt;

&lt;p&gt;Specifically, support rolling back insert, update and delete applyOps operations&lt;br/&gt;
such as those generated by sharding&apos;s moveChunk, splitChunk and mergeChunks&lt;br/&gt;
operations.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/fc81fdee1da1d949f80075c8a88998fa3b0c5e78&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/fc81fdee1da1d949f80075c8a88998fa3b0c5e78&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1124204" author="schwerin" created="Wed, 30 Dec 2015 19:56:06 +0000"  >&lt;p&gt;This failure occurred because an election took place in the config server replica set and switched which node was primary while a chunk split was being performed. The final config server writes performed by chunk split, merge and move operations are bundled into a single operation from the perspective of the replication subsystem. That operation is called &quot;applyOps&quot;, and is only by sharding and certain backup/restore tools. When the election occurred, the applyOps operation in question had replicated to at most half of the voting nodes in the config server replica set, and so was not fully committed. The new primary had not yet received the operation, and so nodes that had were forced to roll it back. Unfortunately, the replication system in 3.2.0 and 3.2.1 does not know how to roll back these operations. We&apos;ll fix this for 3.2.2.&lt;/p&gt;

&lt;p&gt;In the meantime, you could probably reduce your exposure to this type of error by having fewer voting members in your config server replica set. The number of voters you need should be a function of the number of single-node failures you want to tolerate while still allowing chunk migrations, collection create and other metadata operations to proceed. We typically recommend 3 voting nodes, which allows the config servers to keep accepting writes when up to 1 node fails. Since you can keep accepting metadata reads as long as at least 1 config server is up (voting or non-voting), and metadata reads are all you need to accept document reads and writes, this is probably sufficient for most applications.&lt;/p&gt;

&lt;p&gt;Also, when this error happens, you can always remove the data files for the config server that entered this fatal state, and let it resynchronize with the other nodes. Since the config databases are typically small, this should not be a slow process.&lt;/p&gt;</comment>
                            <comment id="1122716" author="ali.hallaji" created="Mon, 28 Dec 2015 07:25:49 +0000"  >&lt;p&gt;Hi Dan,&lt;br/&gt;
We deploy new sharded cluster from scratch.&lt;br/&gt;
We have add 13 members configsvr into replica set, and I setting 6 members into location A and the rest of members into  site B.&lt;br/&gt;
We have 7 members with for voting and other is votes=0 and primary=0.&lt;br/&gt;
Please guide me to deploy strategy for replica set of configsvr for multi data center.&lt;br/&gt;
Can I use from two replica set for configsvr?&lt;/p&gt;</comment>
                            <comment id="1122628" author="dan@10gen.com" created="Mon, 28 Dec 2015 00:18:03 +0000"  >&lt;p&gt;Hi Ali, what is the history of your cluster?  Are you upgrading from v3.0 or creating a new sharded cluster from scratch here?&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>11.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12451" key="com.atlassian.jira.plugin.system.customfieldtypes:multiversion">
                        <customfieldname>Backport Completed</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="15909">3.2.3</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 28 Dec 2015 00:18:03 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        8 years, 2 weeks, 5 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_14262" key="com.atlassian.jira.plugin.system.customfieldtypes:datepicker">
                        <customfieldname>End date</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 4 Jan 2016 23:59:59 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            8 years, 2 weeks, 5 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>Ali.Hallaji</customfieldvalue>
            <customfieldvalue>schwerin@mongodb.com</customfieldvalue>
            <customfieldvalue>dan@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrklk7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hsf5tj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_14261" key="com.atlassian.jira.plugin.system.customfieldtypes:datepicker">
                        <customfieldname>Start date</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 28 Dec 2015 00:00:00 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrpncf:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>