<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:26:08 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-51684] Every 4 hours CPU usage is going high varying from 100% to 200% because of mongo SECONDARY / replica node reciding on server</title>
                <link>https://jira.mongodb.org/browse/SERVER-51684</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;&#160;&lt;br/&gt;
&lt;ins&gt;cat /var/log/mongodb/mongod.log | grep &quot;Successfully connected to mongo1.production.internal:27017&quot; | sort | uniq&lt;/ins&gt;&lt;/p&gt;

&lt;p&gt;2020-10-16T00:06:57.310+0000 I NETWORK &lt;span class=&quot;error&quot;&gt;&amp;#91;LogicalSessionCacheRefresh&amp;#93;&lt;/span&gt; Successfully connected to mongo1.production.internal:27017 (527 connections now open to mongo1.production.internal:27017 with a 0 second timeout)&lt;br/&gt;
2020-10-16T04:06:57.422+0000 I NETWORK &lt;span class=&quot;error&quot;&gt;&amp;#91;LogicalSessionCacheRefresh&amp;#93;&lt;/span&gt; Successfully connected to mongo1.production.internal:27017 (528 connections now open to mongo1.production.internal:27017 with a 0 second timeout)&lt;br/&gt;
2020-10-16T08:06:56.913+0000 I NETWORK &lt;span class=&quot;error&quot;&gt;&amp;#91;LogicalSessionCacheRefresh&amp;#93;&lt;/span&gt; Successfully connected to mongo1.production.internal:27017 (529 connections now open to mongo1.production.internal:27017 with a 0 second timeout)&lt;/p&gt;

&lt;p&gt;&#160;&lt;br/&gt;
&lt;ins&gt;mongod --version&lt;/ins&gt;&lt;/p&gt;

&lt;p&gt;db version v4.0.5&lt;br/&gt;
git version: 3739429dd92b92d1b0ab120911a23d50bf03c412&lt;br/&gt;
OpenSSL version: OpenSSL 1.0.2n  7 Dec 2017&lt;br/&gt;
allocator: tcmalloc&lt;br/&gt;
modules: none&lt;br/&gt;
build environment:&lt;br/&gt;
    distmod: ubuntu1404&lt;br/&gt;
    distarch: x86_64&lt;br/&gt;
    target_arch: x86_64&lt;/p&gt;



&lt;p&gt;&lt;ins&gt;mongo --version&lt;/ins&gt;&lt;/p&gt;

&lt;p&gt;git version: 3739429dd92b92d1b0ab120911a23d50bf03c412&lt;br/&gt;
OpenSSL version: OpenSSL 1.0.2n 7 Dec 2017&lt;br/&gt;
allocator: tcmalloc&lt;br/&gt;
modules: none&lt;br/&gt;
build environment:&lt;br/&gt;
 distmod: ubuntu1404&lt;br/&gt;
 distarch: x86_64&lt;br/&gt;
 target_arch: x86_64&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;</description>
                <environment></environment>
        <key id="1516786">SERVER-51684</key>
            <summary>Every 4 hours CPU usage is going high varying from 100% to 200% because of mongo SECONDARY / replica node reciding on server</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="edwin.zhou@mongodb.com">Edwin Zhou</assignee>
                                    <reporter username="pmakhija@ask-fast.com">Prakhar Makhija</reporter>
                        <labels>
                    </labels>
                <created>Fri, 16 Oct 2020 11:59:48 +0000</created>
                <updated>Wed, 28 Oct 2020 02:59:32 +0000</updated>
                            <resolved>Wed, 28 Oct 2020 02:59:32 +0000</resolved>
                                    <version>4.0.5</version>
                                                    <component>Networking</component>
                    <component>Performance</component>
                    <component>Replication</component>
                                        <votes>0</votes>
                                    <watches>7</watches>
                                                                                                                <comments>
                            <comment id="3464181" author="JIRAUSER1257302" created="Tue, 27 Oct 2020 06:03:40 +0000"  >&lt;p&gt;Hi @edwin.zhou@mongodb.com&lt;/p&gt;

&lt;p&gt;Thanks for sharing the analysis, and guiding me in the right direction&lt;/p&gt;

&lt;p&gt;Will look into it&lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
Prakhar&lt;/p&gt;</comment>
                            <comment id="3461467" author="JIRAUSER1257066" created="Fri, 23 Oct 2020 16:46:08 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=pmakhija%40ask-fast.com&quot; class=&quot;user-hover&quot; rel=&quot;pmakhija@ask-fast.com&quot;&gt;pmakhija@ask-fast.com&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Thank you for sharing your diagnostics with us. After some investigating, we suspect that an external process is increasing your system&apos;s reported CPU load.&lt;/p&gt;

&lt;p&gt;Here is an analysis of the timeline as evidence of this:&lt;/p&gt;

&lt;p&gt;&lt;b&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/284311/284311_Screen+Shot+2020-10-23+at+11.55.14+AM.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/b&gt;&lt;/p&gt;

&lt;p&gt;&lt;b&gt;Point A&lt;/b&gt;&lt;/p&gt;

&lt;p&gt;System memory and CPU usage begins to increase at exactly 00:00 indicated by &lt;tt&gt;system memory anon&lt;/tt&gt;. The &lt;tt&gt;mongod&lt;/tt&gt; does not perform scheduled tasks like this. As system memory increases, memory allocated by the &lt;tt&gt;mongod&lt;/tt&gt; stays unchanged as indicated by&#160;&lt;tt&gt;current_allocated_bytes&lt;/tt&gt;. There is also an increase in CPU iowait, which isn&apos;t exactly CPU usage. Instead, this means that the CPU is needing to wait for i/o, as confirmed by the disk usage. However, looking at &lt;tt&gt;block-manager read/write&lt;/tt&gt;,&#160;the&#160;&lt;tt&gt;mongod&lt;/tt&gt; isn&apos;t doing more i/o.&lt;/p&gt;

&lt;p&gt;&lt;b&gt;Point B&lt;/b&gt;&lt;/p&gt;

&lt;p&gt;Resident memory used by the &lt;tt&gt;mongod&lt;/tt&gt; begins to decrease but allocated memory remains unchanged. We suspect an external process has allocated memory which caused paging and memory used by the &lt;tt&gt;mongod&lt;/tt&gt; is forced out.&lt;/p&gt;

&lt;p&gt;&lt;b&gt;Point C&lt;/b&gt;&lt;br/&gt;
 Memory used by the &lt;tt&gt;mongod&lt;/tt&gt; returns to normal and it appears the external process has completed.&lt;/p&gt;

&lt;p&gt;The &lt;tt&gt;mongod&lt;/tt&gt; does not perform any tasks that occur at an exact time, i.e. exactly at midnight and every 4 hours. Since this appears to be a scheduled task, you should check to see if there are any running cron jobs that may be affecting your system. You can also run &lt;tt&gt;ps&lt;/tt&gt; or &lt;tt&gt;top&lt;/tt&gt; during the times that CPU iowait is high to identify running processes that increase load.&lt;/p&gt;

&lt;p&gt;Best,&lt;br/&gt;
 Edwin&lt;/p&gt;</comment>
                            <comment id="3453969" author="JIRAUSER1257302" created="Tue, 20 Oct 2020 14:16:05 +0000"  >&lt;p&gt;Hi @Bruce Lucas @Eric Sedor&lt;/p&gt;

&lt;p&gt;My bad, the link was not public then, it is now&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://ask-fast-mongo.s3-eu-west-1.amazonaws.com/diagnostic.data.tar.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;diagnostic.data.tar.gz&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;_____________________________________________&lt;/p&gt;


&lt;p&gt;Regarding TimeZones&lt;/p&gt;

&lt;p&gt;I fall in IST (GMT+5:30)&lt;/p&gt;

&lt;p&gt;The SNMP alert email specifies CEST (GMT+2:00)&lt;/p&gt;

&lt;p&gt;The mongod server is residing in Europe (Ireland) - Amazon Region (eu-west-1)&lt;br/&gt;
The mongod server replica node logs are being generated in GMT&lt;/p&gt;

&lt;p&gt;_____________________________________________&lt;/p&gt;


&lt;p&gt;SNMP Email Alerts&lt;/p&gt;

&lt;p&gt;Date/Time: Tue Oct 20 14:09:03 CEST 2020&lt;br/&gt;
CRITICAL - CPU Load (5 min average): 128 %&lt;/p&gt;

&lt;p&gt;Date/Time: Tue Oct 20 10:10:51 CEST 2020&lt;br/&gt;
CRITICAL - CPU Load (5 min average): 162 %&lt;/p&gt;

&lt;p&gt;Date/Time: Tue Oct 20 02:13:28 CEST 2020&lt;br/&gt;
CRITICAL - CPU Load (5 min average): 138 %&lt;/p&gt;

&lt;p&gt;Date/Time: Mon Oct 19 22:10:16 CEST 2020&lt;br/&gt;
CRITICAL - CPU Load (5 min average): 154 %&lt;/p&gt;

&lt;p&gt;Date/Time: Mon Oct 19 18:12:04 CEST 2020&lt;br/&gt;
CRITICAL - CPU Load (5 min average): 182 %&lt;/p&gt;

&lt;p&gt;Date/Time: Mon Oct 19 14:08:52 CEST 2020&lt;br/&gt;
CRITICAL - CPU Load (5 min average): 130 %&lt;/p&gt;

&lt;p&gt;Date/Time: Mon Oct 19 10:10:42 CEST 2020&lt;br/&gt;
CRITICAL - CPU Load (5 min average): 176 %&lt;/p&gt;

&lt;p&gt;Date/Time: Mon Oct 19 06:12:26 CEST 2020&lt;br/&gt;
CRITICAL - CPU Load (5 min average): 170 %&lt;/p&gt;

&lt;p&gt;Date/Time: Mon Oct 19 02:14:10 CEST 2020&lt;br/&gt;
CRITICAL - CPU Load (5 min average): 145 %&lt;/p&gt;

&lt;p&gt;Date/Time: Sun Oct 18 22:10:59 CEST 2020&lt;br/&gt;
CRITICAL - CPU Load (5 min average): 180 %&lt;/p&gt;

&lt;p&gt;Date/Time: Sun Oct 18 18:12:44 CEST 2020&lt;br/&gt;
CRITICAL - CPU Load (5 min average): 208 %&lt;/p&gt;

&lt;p&gt;Date/Time: Sun Oct 18 14:09:30 CEST 2020&lt;br/&gt;
CRITICAL - CPU Load (5 min average): 122 %&lt;/p&gt;

&lt;p&gt;Date/Time: Sun Oct 18 10:11:19 CEST 2020&lt;br/&gt;
CRITICAL - CPU Load (5 min average): 206 %&lt;/p&gt;

&lt;p&gt;Date/Time: Sun Oct 18 06:13:05 CEST 2020&lt;br/&gt;
CRITICAL - CPU Load (5 min average): 164 %&lt;/p&gt;

&lt;p&gt;_____________________________________________&lt;/p&gt;


&lt;p&gt;Server Mongo Logs&lt;/p&gt;

&lt;p&gt; &lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/attachment/283642/283642_mongo-open-connections.out&quot; title=&quot;mongo-open-connections.out attached to SERVER-51684&quot;&gt;mongo-open-connections.out&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.mongodb.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt; &lt;/p&gt;

&lt;p&gt; &lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/attachment/283643/283643_mongod-2020-10-20.log&quot; title=&quot;mongod-2020-10-20.log attached to SERVER-51684&quot;&gt;mongod-2020-10-20.log&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.mongodb.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt; &lt;/p&gt;

&lt;p&gt; &lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/attachment/283645/283645_mongod-2020-10-19.log&quot; title=&quot;mongod-2020-10-19.log attached to SERVER-51684&quot;&gt;mongod-2020-10-19.log&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.mongodb.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt; &lt;/p&gt;</comment>
                            <comment id="3453792" author="bruce.lucas@10gen.com" created="Tue, 20 Oct 2020 12:57:43 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=pmakhija%40ask-fast.com&quot; class=&quot;user-hover&quot; rel=&quot;pmakhija@ask-fast.com&quot;&gt;pmakhija@ask-fast.com&lt;/a&gt;, thanks for collecting the data. Unfortunately the link you provided produces an access denied error when we try to use it.&lt;/p&gt;

&lt;p&gt;Can you please attach the data to this ticket? If it is too large to attach, you can upload it to &lt;a href=&quot;https://10gen-httpsupload.s3.amazonaws.com/upload_forms/dd7aa874-29b6-452c-a814-52815c80abf4.html&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;this secure private upload portal&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;Also, can you please provide a timeline of your issue: which node is seeing the problem at what specific time (including timezone)?&lt;/p&gt;</comment>
                            <comment id="3453572" author="JIRAUSER1257302" created="Tue, 20 Oct 2020 10:38:17 +0000"  >&lt;p&gt;Hi @eric.sedor , please find the tarball attached&lt;/p&gt;

&lt;p&gt;&#160;&lt;a href=&quot;https://ask-fast-mongo.s3-eu-west-1.amazonaws.com/diagnostic.data.tar.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;diagnostic.data.tar.gz&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Feel free to let me know if anything else is needed&lt;/p&gt;


&lt;p&gt;Thanks,&lt;/p&gt;

&lt;p&gt;Prakhar&lt;/p&gt;</comment>
                            <comment id="3453118" author="eric.sedor" created="Mon, 19 Oct 2020 23:08:51 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=pmakhija%40ask-fast.com&quot; class=&quot;user-hover&quot; rel=&quot;pmakhija@ask-fast.com&quot;&gt;pmakhija@ask-fast.com&lt;/a&gt;, can you please provide more information about the issue you are reporting? High CPU use could have many causes.&lt;/p&gt;

&lt;p&gt;As well, would you please archive (tar or zip) the &lt;tt&gt;$dbpath/diagnostic.data&lt;/tt&gt; directory (the contents are described &lt;a href=&quot;https://docs.mongodb.com/manual/administration/analyzing-mongodb-performance/#full-time-diagnostic-data-capture&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;here&lt;/a&gt;) and attach it to this ticket?&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Eric&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="284311" name="Screen Shot 2020-10-23 at 11.55.14 AM.png" size="351554" author="edwin.zhou@mongodb.com" created="Fri, 23 Oct 2020 19:16:07 +0000"/>
                            <attachment id="283642" name="mongo-open-connections.out" size="119999" author="pmakhija@ask-fast.com" created="Tue, 20 Oct 2020 14:11:32 +0000"/>
                            <attachment id="283645" name="mongod-2020-10-19.log" size="104619609" author="pmakhija@ask-fast.com" created="Tue, 20 Oct 2020 14:15:12 +0000"/>
                            <attachment id="283643" name="mongod-2020-10-20.log" size="44098204" author="pmakhija@ask-fast.com" created="Tue, 20 Oct 2020 14:13:17 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>6.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 19 Oct 2020 23:08:51 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        3 years, 15 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>edwin.zhou@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            3 years, 15 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>edwin.zhou@mongodb.com</customfieldvalue>
            <customfieldvalue>eric.sedor@mongodb.com</customfieldvalue>
            <customfieldvalue>pmakhija@ask-fast.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hyd34f:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hxzcfr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[edwin.zhou@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hycpdr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>