<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:51:24 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-39219] Insert Performance with MongoDB 3.4.10 and MongoDB 4.0.5 by mongostat</title>
                <link>https://jira.mongodb.org/browse/SERVER-39219</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;When we insert&#160;700 million data with MongoDB 3.4.10, it needs 4.5 hours. But when we upgrade to MongoDB 4.0.5, it needs 6.5 hours. As time goes on, insert the same data need more and more time. Where can I check the setting to improve my performance.&lt;/p&gt;

&lt;p&gt;In MongoDB 3.4.10 cache dirty under 6%, but in MongoDB 4.0.5 cache dirty will up to 20%. We install MongoDB with default setting, and disable NUMA. Related content as attachments, thanks a lot!!&lt;/p&gt;</description>
                <environment></environment>
        <key id="678736">SERVER-39219</key>
            <summary>Insert Performance with MongoDB 3.4.10 and MongoDB 4.0.5 by mongostat</summary>
                <type id="6" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14720&amp;avatarType=issuetype">Question</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13203">Gone away</resolution>
                                        <assignee username="backlog-server-storage-engines">Backlog - Storage Engines Team</assignee>
                                    <reporter username="dennischien">Dennis</reporter>
                        <labels>
                            <label>customer-mgmt</label>
                    </labels>
                <created>Mon, 28 Jan 2019 09:42:50 +0000</created>
                <updated>Fri, 27 Oct 2023 20:43:01 +0000</updated>
                            <resolved>Mon, 13 Mar 2023 03:08:11 +0000</resolved>
                                    <version>4.0.5</version>
                                    <fixVersion>4.3 Desired</fixVersion>
                                    <component>Performance</component>
                    <component>Storage</component>
                    <component>WiredTiger</component>
                                        <votes>0</votes>
                                    <watches>15</watches>
                                                                                                                <comments>
                            <comment id="5266526" author="sulabh.mahajan" created="Mon, 13 Mar 2023 03:05:22 +0000"  >&lt;p&gt;I am closing this issue as it is no longer relevant for the currently supported releases. Please re-open a new ticket if this applies to a more recent release.&lt;/p&gt;</comment>
                            <comment id="2398676" author="dennischien" created="Thu, 29 Aug 2019 01:27:18 +0000"  >&lt;p&gt;Hi Bruce,&lt;/p&gt;

&lt;p&gt;We are glad to inform you that we updated the information for the issue. We observe that memory grows up, INSERT data slow down. However, the MongoDB service restarts, the insert operation returns to be normal. Please refer to the attachment for the Ops Manager information. &lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Dennis&lt;/p&gt;</comment>
                            <comment id="2224271" author="dennischien" created="Thu, 25 Apr 2019 05:47:51 +0000"  >&lt;p&gt;Hi Bruce,&lt;/p&gt;

&lt;p&gt;Thanks for your assistance, if there has any question, please don&apos;t hesitate to let me know. Thanks.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Dennis&lt;/p&gt;</comment>
                            <comment id="2223066" author="bruce.lucas@10gen.com" created="Wed, 24 Apr 2019 12:08:22 +0000"  >&lt;p&gt;Hi Dennis,&lt;/p&gt;

&lt;p&gt;Sorry for the delay. Unfortunately we don&apos;t have any recommendations yet. The cache.png attachment reflects some internal investigation we are doing with the storage engine team; we&apos;ll update you with the outcome of that discussion.&lt;/p&gt;

&lt;p&gt;Bruce&lt;/p&gt;</comment>
                            <comment id="2222827" author="dennischien" created="Wed, 24 Apr 2019 02:28:50 +0000"  >&lt;p&gt;Hi Bruce,&lt;/p&gt;

&lt;p&gt;Thanks for the updated comment and providing that data.&lt;br/&gt;
According the cache.png, can you give me some suggestions? Please feel free to tell me if there is anything setting need to change. Thank you. &lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Dennis&lt;/p&gt;</comment>
                            <comment id="2212876" author="dennischien" created="Mon, 15 Apr 2019 01:19:44 +0000"  >&lt;p&gt;Hi Daniel,&lt;/p&gt;

&lt;p&gt;Sorry to trouble you again!&lt;br/&gt;
If you have any suggestions for improvement, please contact me. Thank you.&lt;/p&gt;


&lt;p&gt;Thanks,&lt;br/&gt;
Dennis&lt;/p&gt;</comment>
                            <comment id="2188327" author="dennischien" created="Fri, 22 Mar 2019 08:36:23 +0000"  >&lt;p&gt;Hi Danny,&lt;/p&gt;

&lt;p&gt;Thanks for the updated comment.&lt;br/&gt;
We have run 3.4.10 case again and uploaded new diagnostic.data ( field name: diagnostic.data_3.4.10_PSS_20190322.zip ) at the private link you provided. Let me know if you find anything. Thanks a lot!&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Dennis&lt;/p&gt;</comment>
                            <comment id="2185310" author="daniel.hatcher" created="Tue, 19 Mar 2019 20:22:15 +0000"  >&lt;p&gt;Hello Dennis,&lt;/p&gt;

&lt;p&gt;I apologize for not responding earlier. It appears that we are still lacking the diagnostic.data for 3.4.10 second test on the Secondary. However, it may not necessary if it does not exist. We are still investigating the issue and will reach out to you again soon.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;/p&gt;

&lt;p&gt;Danny&lt;/p&gt;</comment>
                            <comment id="2184561" author="dennischien" created="Tue, 19 Mar 2019 02:22:07 +0000"  >&lt;p&gt;Hi Bruce,&lt;/p&gt;

&lt;p&gt;I am just wondering if you have received my previous attachment. Sorry for any inconvenience that could cause. Let me know if you need anything else. Thanks a lot!&lt;/p&gt;

&lt;p&gt;Best,&lt;br/&gt;
Dennis&lt;/p&gt;</comment>
                            <comment id="2175490" author="dennischien" created="Fri, 8 Mar 2019 08:38:38 +0000"  >&lt;p&gt;Hi Daniel,&lt;/p&gt;

&lt;p&gt;Thanks for the updated comment.&lt;br/&gt;
As requested in your comment, I have re-uploaded the diagnostic.data ( diagnostic.data_3.4.10_PSS_20190308.zip, diagnostic.data_4.0.5_PSS_201900308.zip ) at the private link you provided. Let me know if you need anything else. Thank you again for your attention.&lt;/p&gt;

&lt;p&gt;Best,&lt;br/&gt;
Dennis&lt;/p&gt;
</comment>
                            <comment id="2175218" author="daniel.hatcher" created="Thu, 7 Mar 2019 22:33:14 +0000"  >&lt;p&gt;Hello Dennis,&lt;/p&gt;

&lt;p&gt;Thanks for your patience, we are still looking into this. I do see that the most recent uploads do not include Secondary diagnostic.data for the 3.4.10 second test run. Do you still have those files available?&lt;/p&gt;

&lt;p&gt;Danny&lt;/p&gt;</comment>
                            <comment id="2163819" author="dennischien" created="Wed, 27 Feb 2019 00:51:44 +0000"  >&lt;p&gt;Hi Bruce,&lt;/p&gt;

&lt;p&gt;Sorry to trouble you again!&lt;br/&gt;
I have uploaded our diagnostic.data ( diagnostic.data_3.4.10_PSS_20190220.zip, diagnostic.data_4.0.5_PSS_20190220.zip ) at the private link you provided. If you have any suggestions for improvement, please do not hesitate to contact me. Looking forward to your favorable response.&lt;/p&gt;

&lt;p&gt;Best,&lt;br/&gt;
Dennis&lt;/p&gt;</comment>
                            <comment id="2156705" author="dennischien" created="Wed, 20 Feb 2019 06:53:48 +0000"  >&lt;p&gt;Hi Bruce, &lt;/p&gt;

&lt;p&gt;Thanks for the updated comment.&lt;/p&gt;

&lt;p&gt;As requested in your comment, I have re-uploaded the diagnostic.data ( diagnostic.data_3.4.10_PSS_20190220.zip, diagnostic.data_4.0.5_PSS_20190220.zip ) at the private link you provided. &lt;br/&gt;
Thank you again for your attention.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Dennis&lt;/p&gt;</comment>
                            <comment id="2156190" author="bruce.lucas@10gen.com" created="Tue, 19 Feb 2019 20:47:37 +0000"  >&lt;p&gt;Hi Dennis,&lt;/p&gt;

&lt;p&gt;Thanks for uploading the new data.&lt;/p&gt;

&lt;p&gt;In the 4.0.5 case we see the secondaries sometimes lagging, sometimes significantly (tens of minutes). In 4.0 high lag creates additional cache pressure on the primary that can slow it down.&lt;/p&gt;

&lt;p&gt;I would like to investigate why the lag is occurring in 4.0.5 but not 3.4.10. Can you please upload the diagnostic.data directories from both secondaries covering both of those tests?&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Bruce&lt;/p&gt;</comment>
                            <comment id="2155082" author="dennischien" created="Tue, 19 Feb 2019 02:13:41 +0000"  >&lt;p&gt;Hi Bruce,&lt;/p&gt;

&lt;p&gt;I am just wondering if you have received my previous attachment. &#160;I am uploaded it again just in case you might not have got it yet. Sorry for any inconvenience that could cause.&#160;Let me know if you need anything else. Thanks a lot!&#160;Looking forward to your favorable response.&lt;/p&gt;



&lt;p&gt;Best,&lt;br/&gt;
Dennis&lt;/p&gt;</comment>
                            <comment id="2152622" author="dennischien" created="Fri, 15 Feb 2019 09:00:36 +0000"  >&lt;p&gt;Hi Bruce,&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Thanks for the updated information.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;I&apos;ll try to answer the questions as much as possible:&lt;/p&gt;

&lt;p&gt;We run 3.4.10 case and 4.0.5 case on the same host with the same network environment. In the same condition, there is great difference between execution time and performance.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Test data information:&lt;/p&gt;

&lt;p&gt;Data count: 600 million&lt;/p&gt;

&lt;p&gt;Storage size: 320 GB&lt;/p&gt;
&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&#160;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;MongoDB 3.4.10&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;MongoDB 4.0.5&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;First Time&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;5 hours&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;5 hours&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Second Time&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;5 hours&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;7 hours&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Remarks&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Whether the MongoDB service is restarted or not, it is maintained for 5 hours.&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;As time goes on, the execution time needs more and more. When the MongoDB service restarts, the execution time returns to 5 hours.&lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;


&lt;p&gt;&#160;&#160;&lt;/p&gt;

&lt;p&gt;Beside, as you stated before, in the 4.0.5 case we see one secondary member out of sync, so the information in the 4.0.5 case was incorrect. I have re-uploaded the diagnostic.data ( diagnostic.data_3.4.10_20190215.zip, diagnostic.data_4.0.5_20190215.zip ) which was retested at the private link you provided. Could you please help us to re-diagnose the cause?&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Thanks. Let me know if you need anything else.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Dennis&lt;/p&gt;</comment>
                            <comment id="2133411" author="bruce.lucas@10gen.com" created="Thu, 31 Jan 2019 15:37:50 +0000"  >&lt;p&gt;Hi Dennis,&lt;/p&gt;

&lt;p&gt;In the 4.0.5 case we see one secondary member lagging severely until lag built to more than an hour and it fell off the oplog. A severely lagging secondary can result in higher cache pressure and lower performance on the node that the secondary is syncing from because old parts of the oplog must be kept in cache or re-read into cache, and indeed we see a high rate of data being read into cache for the oplog in the 4.0.5 case.&lt;/p&gt;

&lt;p&gt;In the 3.4.10 case we also see a lagging secondary, but we don&apos;t see a high rate of oplog data being read into cache. This is likely because the lagging secondary was not syncing from the primary in that case but rather from the other secondary.&lt;/p&gt;

&lt;p&gt;In the 4.0.5 data we also see a high rate of tcp retransmissions. (We don&apos;t record that information in 3.4 until 3.4.16). This makes me suspect the lag is likely related to a network issue. I&apos;d recommend that you investigate the cause of the lag, and the possible network issue.&lt;/p&gt;

&lt;p&gt;Bruce&lt;/p&gt;</comment>
                            <comment id="2132905" author="dennischien" created="Thu, 31 Jan 2019 01:53:29 +0000"  >&lt;p&gt;Hey Bruce,&lt;/p&gt;

&lt;p&gt;I have uploaded the diagnostic.data(diagnostic.data_3.4.10.zip, diagnostic.data_4.0.5.zip) and the mongostat log at the private link you provided. Let me know if you find anything. &lt;img class=&quot;emoticon&quot; src=&quot;https://jira.mongodb.org/images/icons/emoticons/smile.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;br/&gt;
 Thanks a lot!&lt;/p&gt;

&lt;p&gt;Best,&lt;br/&gt;
 Dennis&lt;/p&gt;</comment>
                            <comment id="2130429" author="bruce.lucas@10gen.com" created="Tue, 29 Jan 2019 13:15:53 +0000"  >&lt;p&gt;Hi Dennis,&lt;/p&gt;

&lt;p&gt;We don&apos;t at this point provide a viewer for the diagnostic data. The data collected in the diagnostic.data directory is described &lt;a href=&quot;https://docs.mongodb.com/master/administration/analyzing-mongodb-performance/#full-time-diagnostic-data-capture&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;here&lt;/a&gt;. Also, you may view the code that collects that data &lt;a href=&quot;https://github.com/mongodb/mongo/tree/master/src/mongo/db/ftdc&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;here&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;You can upload your files to &lt;a href=&quot;https://10gen-httpsupload.s3.amazonaws.com/upload_forms/898c24a8-ed8a-4df1-90c5-f8eb6d7cf734.html&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;this&lt;/a&gt; secure private portal. This will also allow you to upload the entire diagnostic.data directory, which may exceed the JIRA attachment limit.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Bruce&lt;/p&gt;</comment>
                            <comment id="2130247" author="dennischien" created="Tue, 29 Jan 2019 07:30:12 +0000"  >&lt;p&gt;Hi Bruce,&lt;/p&gt;

&lt;p&gt;Thanks for adding the comment.&#160;&lt;/p&gt;

&lt;p&gt;The files in diagnostic.data directory look like binary data. Could you please tell me how to view this data? We&#160;don&apos;t want to unknowingly share any secret data.&#160;&#160;The company has strict policy regarding sharing of such data.&#160;Or where can we upload to share the information with the assurance that keep it private?&#160;&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Dennis&lt;/p&gt;</comment>
                            <comment id="2129011" author="bruce.lucas@10gen.com" created="Mon, 28 Jan 2019 13:28:42 +0000"  >&lt;p&gt;Hi Dennis,&lt;/p&gt;

&lt;p&gt;Can you please archive and attach the content of the $dbpath/diagnostic.data directory? It is important that the diagnostic data should cover both the 3.4 and 4.0 insert jobs; you can check the time period covered by looking at the metrics.* filenames, which are timestamps indicating the start of the data covered by that file. Also, if you can attach mongod log files covering the two import jobs that may be helfpul.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Bruce&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="207316" name="3.4.10.txt" size="35809" author="dennischien" created="Mon, 28 Jan 2019 09:40:20 +0000"/>
                            <attachment id="207315" name="4.0.5.txt" size="23416" author="dennischien" created="Mon, 28 Jan 2019 09:40:27 +0000"/>
                            <attachment id="214657" name="cache.png" size="276339" author="bruce.lucas@mongodb.com" created="Fri, 19 Apr 2019 14:28:20 +0000"/>
                            <attachment id="228870" name="ops_manager_20190829.png" size="89854" author="dennischien" created="Thu, 29 Aug 2019 01:02:56 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>21.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>3.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_12751" key="com.atlassian.jira.plugin.system.customfieldtypes:multiselect">
                        <customfieldname>Assigned Teams</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="25135"><![CDATA[Storage Engines]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 28 Jan 2019 13:28:42 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        47 weeks, 3 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            47 weeks, 3 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>backlog-server-storage-engines</customfieldvalue>
            <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>daniel.hatcher@mongodb.com</customfieldvalue>
            <customfieldvalue>dennischien</customfieldvalue>
            <customfieldvalue>sulabh.mahajan@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hukddb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hxxz0v:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="3039">Storage Engines 2019-06-17</customfieldvalue>
    <customfieldvalue id="3058">Storage Engines 2019-07-01</customfieldvalue>
    <customfieldvalue id="3115">Storage Engines 2019-08-12</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10555" key="com.atlassian.jira.plugin.system.customfieldtypes:float">
                        <customfieldname>Story Points</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>5.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[bruce.lucas@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hujzmn:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>