<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:30:19 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-32465] Abnormally high (100%) CPU on secondary instance</title>
                <link>https://jira.mongodb.org/browse/SERVER-32465</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;We experienced the following on our production replica set: 3 secondaries 1 primary, MongoDB 3.2.12 + WiredTiger.&lt;/p&gt;

&lt;p&gt;1. All secondaries saw a spike in disk usage (about 4-5x base load, we see something like this about twice per day randomly)&lt;br/&gt;
2. One of the secondary&apos;s mongod CPU usage shot up to 100%. At this time the replica set continued to route queries to the affected instance, but the instance did not process them.&lt;br/&gt;
3. We killed all queries on the affected instance via console, the CPU remained at 100% it continued to NOT process queries.&lt;br/&gt;
4. Finally we removed the affected process from the replica set and again we could process all queries.&lt;/p&gt;

&lt;p&gt;Here is a CPU chart of the 4 instances. As you can see CPU of one of the instances hit 100%. (As it was not responding to queries, our app failed and stopped sending new queries, so other instances went to low CPU)&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;a id=&quot;174316_thumb&quot; href=&quot;https://jira.mongodb.org/secure/attachment/174316/174316_image-2017-12-27-19-58-29-795.png&quot; title=&quot;image-2017-12-27-19-58-29-795.png&quot; file-preview-type=&quot;image&quot; file-preview-id=&quot;174316&quot; file-preview-title=&quot;image-2017-12-27-19-58-29-795.png&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/thumbnail/174316/_thumb_174316.png&quot; style=&quot;border: 0px solid black&quot; role=&quot;presentation&quot;/&gt;&lt;/a&gt;&lt;/span&gt; &lt;/p&gt;

&lt;p&gt;Here is a chart showing the disk usage spike on the affected instance which happened before the high CPU.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;a id=&quot;174315_thumb&quot; href=&quot;https://jira.mongodb.org/secure/attachment/174315/174315_image-2017-12-27-19-58-33-746.png&quot; title=&quot;image-2017-12-27-19-58-33-746.png&quot; file-preview-type=&quot;image&quot; file-preview-id=&quot;174315&quot; file-preview-title=&quot;image-2017-12-27-19-58-33-746.png&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/thumbnail/174315/_thumb_174315.png&quot; style=&quot;border: 0px solid black&quot; role=&quot;presentation&quot;/&gt;&lt;/a&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;Unfortunately we did not get a core dump of the affected process.&lt;/p&gt;</description>
                <environment></environment>
        <key id="476370">SERVER-32465</key>
            <summary>Abnormally high (100%) CPU on secondary instance</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13203">Gone away</resolution>
                                        <assignee username="kelsey.schubert@mongodb.com">Kelsey Schubert</assignee>
                                    <reporter username="johnnyshields">Johnny Shields</reporter>
                        <labels>
                    </labels>
                <created>Wed, 27 Dec 2017 11:08:11 +0000</created>
                <updated>Fri, 27 Oct 2023 20:43:44 +0000</updated>
                            <resolved>Fri, 16 Feb 2018 19:54:26 +0000</resolved>
                                    <version>3.2.12</version>
                                                                        <votes>0</votes>
                                    <watches>9</watches>
                                                                                                                <comments>
                            <comment id="1808421" author="thomas.schubert" created="Fri, 16 Feb 2018 19:54:26 +0000"  >&lt;p&gt;Thanks for the update, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=johnnyshields&quot; class=&quot;user-hover&quot; rel=&quot;johnnyshields&quot;&gt;johnnyshields&lt;/a&gt;. Given the infrequency of this issue, I&apos;m going to close this ticket. If you encounter it again on 3.4, please let us know and we can continue to investigate.&lt;/p&gt;</comment>
                            <comment id="1795590" author="johnnyshields" created="Mon, 5 Feb 2018 19:21:36 +0000"  >&lt;p&gt;Upgraded to 3.4 and no issue so far, but its very rare (happened once in 2+ years of running MongoDB 3.2)&lt;/p&gt;</comment>
                            <comment id="1795388" author="thomas.schubert" created="Mon, 5 Feb 2018 17:01:19 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=johnnyshields&quot; class=&quot;user-hover&quot; rel=&quot;johnnyshields&quot;&gt;johnnyshields&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Upgrading to a larger host may resolve this issue, however since we haven&apos;t completed an RCA, I cannot guarantee that it will reduce the likelihood of a recurrence. Would you please let us know if this is still an issue for you after upgrading to the latest version of MongoDB 3.4?&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Kelsey&lt;/p&gt;</comment>
                            <comment id="1765763" author="johnnyshields" created="Fri, 5 Jan 2018 04:25:19 +0000"  >&lt;p&gt;Hi Kelsey, thank you for the clarification and understood. For avoidance of doubt, upgrading to a host with more CPUs (e.g. 4) would be a workaround that would reduce likelihood of recurrence?&lt;/p&gt;</comment>
                            <comment id="1765754" author="thomas.schubert" created="Fri, 5 Jan 2018 03:48:53 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=johnnyshields&quot; class=&quot;user-hover&quot; rel=&quot;johnnyshields&quot;&gt;johnnyshields&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;I&apos;m familiar with &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-30189&quot; title=&quot;Reduce calls to allocator for large $in expressions&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-30189&quot;&gt;&lt;del&gt;SERVER-30189&lt;/del&gt;&lt;/a&gt; and can confirm that it affects MongoDB 3.2. In my previous comment, I suggested that you upgrade to the latest version of MongoDB 3.4 to take advantage of this fix as a number of the queries recorded in your logs have large $in expressions.&lt;/p&gt;

&lt;p&gt;Given the relatively small size of this host (2 cpus), it is more susceptible to connection escalations if the application sends queries faster than it can process. &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-30189&quot; title=&quot;Reduce calls to allocator for large $in expressions&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-30189&quot;&gt;&lt;del&gt;SERVER-30189&lt;/del&gt;&lt;/a&gt; is a known issue that may cause throughput to decrease unexpectedly resulting in this type of behavior. That said, there is not enough information at this time to determine whether or not upgrading will resolve this issue.&lt;/p&gt;

&lt;p&gt;However, I believe upgrading is the best way to progress this investigation for two reasons. First, upgrading to 3.4.10, will allow us to quickly determine whether this fix (which we have identified specifically as affecting your workload) resolves this issue. Second, MongoDB 3.4 contains a number of other performance improvements as well additional diagnostics, which would enable us to better understand what is going on if the issue was not resolved. To that end, I included instruction in my previous comment about the type of diagnostic information we would need if issue continued after upgrading.&lt;/p&gt;

&lt;p&gt;Kind regards,&lt;br/&gt;
Kelsey&lt;/p&gt;</comment>
                            <comment id="1763664" author="johnnyshields" created="Wed, 3 Jan 2018 08:07:38 +0000"  >&lt;p&gt;Hi Kelsey, one more question. I browsed the code differences and it seems that there&apos;s been quite a bit of refactoring in this area between 3.2 and 3.4. It would be helpful if someone who is very familiar with the code and refactoring done can comment if this issue can in fact occur on 3.2?&lt;/p&gt;</comment>
                            <comment id="1763653" author="johnnyshields" created="Wed, 3 Jan 2018 07:32:27 +0000"  >&lt;p&gt;Hi Kelsey, is it correct to say that it appears due to the underlying issue of &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-30189&quot; title=&quot;Reduce calls to allocator for large $in expressions&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-30189&quot;&gt;&lt;del&gt;SERVER-30189&lt;/del&gt;&lt;/a&gt; the affected instance had all threads go into a &quot;dead spin-lock&quot; state (similar to a deadlock but with CPU being consumed)?&lt;/p&gt;

&lt;p&gt;Also, its not entirely clear to me how &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-30189&quot; title=&quot;Reduce calls to allocator for large $in expressions&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-30189&quot;&gt;&lt;del&gt;SERVER-30189&lt;/del&gt;&lt;/a&gt; definitively fixes the issue.&lt;/p&gt;

&lt;p&gt;We will try to reproduce the issue by bombarding the instance with $in queries.&lt;/p&gt;</comment>
                            <comment id="1763163" author="thomas.schubert" created="Tue, 2 Jan 2018 19:09:55 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=johnnyshields&quot; class=&quot;user-hover&quot; rel=&quot;johnnyshields&quot;&gt;johnnyshields&lt;/a&gt; and &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=MiLk&quot; class=&quot;user-hover&quot; rel=&quot;MiLk&quot;&gt;MiLk&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Thanks for providing diagnostic.data and logs. After reviewing the logs, I see a few very large $in queries which occur around the event. Would you please upgrade to the latest version of MongoDB 3.4 to take advantage of &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-30189&quot; title=&quot;Reduce calls to allocator for large $in expressions&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-30189&quot;&gt;&lt;del&gt;SERVER-30189&lt;/del&gt;&lt;/a&gt; and let us know if the issue is resolved?&lt;/p&gt;

&lt;p&gt;If the issue is not resolved after upgrading, please collect perf during the event:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;# record call stack samples and generate text output on test node&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;perf record  -a -g -F 99 sleep 60&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;perf script &amp;gt;perf.txt&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;We&apos;ll want diagnostic.data, logs, and the perf.txt file. Note that in MongoDB 3.4, diagnostic.data also contains disk and cpu utilization metrics, which also help us better understand what is going on here.&lt;/p&gt;

&lt;p&gt;Thanks again for you help,&lt;br/&gt;
Kelsey&lt;/p&gt;</comment>
                            <comment id="1760204" author="milk" created="Wed, 27 Dec 2017 17:40:04 +0000"  >&lt;p&gt;Hi Kelsey,&lt;/p&gt;

&lt;p&gt;I&apos;ve uploaded the diagnostic data from the 4 servers of the replica set, and the logs for the hour around the incident (around 09:33 UTC).&lt;br/&gt;
The primary is &lt;em&gt;mongodb001-05f4&lt;/em&gt; and the affected server is &lt;em&gt;mongodb001-04db&lt;/em&gt;.&lt;/p&gt;

&lt;p&gt;Please let me know if you need additional information.&lt;/p&gt;</comment>
                            <comment id="1759980" author="thomas.schubert" created="Wed, 27 Dec 2017 12:17:45 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=johnnyshields&quot; class=&quot;user-hover&quot; rel=&quot;johnnyshields&quot;&gt;johnnyshields&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Thanks for reporting this issue. So we can continue to investigate, would you please upload the following information?&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;complete mongod logs from the primary, affected secondary, and an unaffected secondary&lt;/li&gt;
	&lt;li&gt;an archive of the diagnostic.data directory from the primary, affected secondary, and an unaffected secondary&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Thank you,&lt;br/&gt;
Kelsey&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="174337" name="2017122709_mongod.log.gz" size="3073923" author="MiLk" created="Wed, 27 Dec 2017 16:56:51 +0000"/>
                            <attachment id="174370" name="Screen Shot 2017-12-27 at 16.25.29.png" size="282900" author="MiLk" created="Thu, 28 Dec 2017 07:15:04 +0000"/>
                            <attachment id="174369" name="Screen Shot 2017-12-27 at 16.25.56.png" size="344404" author="MiLk" created="Thu, 28 Dec 2017 07:15:03 +0000"/>
                            <attachment id="174368" name="Screen Shot 2017-12-27 at 16.26.52.png" size="482289" author="MiLk" created="Thu, 28 Dec 2017 07:14:58 +0000"/>
                            <attachment id="174367" name="Screen Shot 2017-12-27 at 16.27.22.png" size="631945" author="MiLk" created="Thu, 28 Dec 2017 07:15:05 +0000"/>
                            <attachment id="174329" name="diagnostic_data_04db.tar.gz" size="98919815" author="MiLk" created="Wed, 27 Dec 2017 16:09:34 +0000"/>
                            <attachment id="174335" name="diagnostic_data_05f4.tar.gz" size="106152008" author="MiLk" created="Wed, 27 Dec 2017 16:51:59 +0000"/>
                            <attachment id="174334" name="diagnostic_data_0ddb.tar.gz" size="110370348" author="MiLk" created="Wed, 27 Dec 2017 16:51:25 +0000"/>
                            <attachment id="174332" name="diagnostic_data_0f6a.tar.gz" size="107807467" author="MiLk" created="Wed, 27 Dec 2017 16:49:34 +0000"/>
                            <attachment id="174316" name="image-2017-12-27-19-58-29-795.png" size="43442" author="johnnyshields" created="Wed, 27 Dec 2017 10:58:33 +0000"/>
                            <attachment id="174315" name="image-2017-12-27-19-58-33-746.png" size="148258" author="johnnyshields" created="Wed, 27 Dec 2017 10:58:37 +0000"/>
                            <attachment id="174523" name="mongoparsed.log" size="27711210" author="mark.agarunov" created="Tue, 2 Jan 2018 15:23:01 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>10.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 27 Dec 2017 12:17:45 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        5 years, 51 weeks, 5 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            5 years, 51 weeks, 5 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>MiLk</customfieldvalue>
            <customfieldvalue>johnnyshields</customfieldvalue>
            <customfieldvalue>kelsey.schubert@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htmx4f:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hteh7z:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[kelsey.schubert@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htmj8v:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>