<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:16:30 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-48221] Shut down ftdc after storage engine</title>
                <link>https://jira.mongodb.org/browse/SERVER-48221</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;We&apos;ve encountered cases of issues in WT shutdown like &lt;a href=&quot;https://jira.mongodb.org/browse/WT-6164&quot; title=&quot;Reduce the number of page flags according to the flag variable size&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-6164&quot;&gt;&lt;del&gt;WT-6164&lt;/del&gt;&lt;/a&gt; where diagnosis was difficult (and likely impossible in the field) because ftdc is shut down before the storage engine. We should move ftdc shutdown after storage engine shutdown.&lt;/p&gt;</description>
                <environment></environment>
        <key id="1351584">SERVER-48221</key>
            <summary>Shut down ftdc after storage engine</summary>
                <type id="4" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14710&amp;avatarType=issuetype">Improvement</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="gregory.wlodarek@mongodb.com">Gregory Wlodarek</assignee>
                                    <reporter username="bruce.lucas@mongodb.com">Bruce Lucas</reporter>
                        <labels>
                    </labels>
                <created>Thu, 14 May 2020 16:54:39 +0000</created>
                <updated>Sun, 29 Oct 2023 22:08:11 +0000</updated>
                            <resolved>Fri, 13 Nov 2020 03:00:20 +0000</resolved>
                                                    <fixVersion>4.9.0</fixVersion>
                    <fixVersion>4.4.3</fixVersion>
                                    <component>Diagnostics</component>
                                        <votes>0</votes>
                                    <watches>16</watches>
                                                                                                                <comments>
                            <comment id="3527466" author="xgen-internal-githook" created="Mon, 14 Dec 2020 05:02:26 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Gregory Wlodarek&apos;, &apos;email&apos;: &apos;gregory.wlodarek@mongodb.com&apos;, &apos;username&apos;: &apos;GWlodarek&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-48221&quot; title=&quot;Shut down ftdc after storage engine&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-48221&quot;&gt;&lt;del&gt;SERVER-48221&lt;/del&gt;&lt;/a&gt; Shut down ftdc after storage engine&lt;/p&gt;

&lt;p&gt;(cherry picked from commit 185000ad894d5cb95a3c946158712054db57cb7a)&lt;br/&gt;
Branch: v4.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/73267a2a82d164a3457805c80e2173a5a4f1db60&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/73267a2a82d164a3457805c80e2173a5a4f1db60&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3527464" author="xgen-internal-githook" created="Mon, 14 Dec 2020 05:02:16 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Gregory Wlodarek&apos;, &apos;email&apos;: &apos;gregory.wlodarek@mongodb.com&apos;, &apos;username&apos;: &apos;GWlodarek&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-48221&quot; title=&quot;Shut down ftdc after storage engine&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-48221&quot;&gt;&lt;del&gt;SERVER-48221&lt;/del&gt;&lt;/a&gt; Shut down ftdc after storage engine&lt;/p&gt;

&lt;p&gt;(cherry picked from commit f7274c2082f729c1715b2dfd7fd233016a2a58a6)&lt;br/&gt;
Branch: v4.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/10gen/mongo-enterprise-modules/commit/15a7aca8711be9f9009c17f26d65bd734e7f488c&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/10gen/mongo-enterprise-modules/commit/15a7aca8711be9f9009c17f26d65bd734e7f488c&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3488723" author="xgen-internal-githook" created="Fri, 13 Nov 2020 01:14:05 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Gregory Wlodarek&apos;, &apos;email&apos;: &apos;gregory.wlodarek@mongodb.com&apos;, &apos;username&apos;: &apos;GWlodarek&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-48221&quot; title=&quot;Shut down ftdc after storage engine&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-48221&quot;&gt;&lt;del&gt;SERVER-48221&lt;/del&gt;&lt;/a&gt; Shut down ftdc after storage engine&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/185000ad894d5cb95a3c946158712054db57cb7a&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/185000ad894d5cb95a3c946158712054db57cb7a&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3488722" author="xgen-internal-githook" created="Fri, 13 Nov 2020 01:13:59 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Gregory Wlodarek&apos;, &apos;email&apos;: &apos;gregory.wlodarek@mongodb.com&apos;, &apos;username&apos;: &apos;GWlodarek&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-48221&quot; title=&quot;Shut down ftdc after storage engine&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-48221&quot;&gt;&lt;del&gt;SERVER-48221&lt;/del&gt;&lt;/a&gt; Shut down ftdc after storage engine&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/10gen/mongo-enterprise-modules/commit/f7274c2082f729c1715b2dfd7fd233016a2a58a6&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/10gen/mongo-enterprise-modules/commit/f7274c2082f729c1715b2dfd7fd233016a2a58a6&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3487478" author="gregory.wlodarek" created="Thu, 12 Nov 2020 14:52:48 +0000"  >&lt;p&gt;I&apos;ve split off the &lt;b&gt;rollback_to_stable&lt;/b&gt; before&#160;&lt;b&gt;WT_CONNECTION::close&lt;/b&gt;&#160;work into &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-52815&quot; title=&quot;Investigate calling &amp;#39;rollback_to_stable&amp;#39; prior to shutting down WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-52815&quot;&gt;SERVER-52815&lt;/a&gt; to make it easier to review the two changes separately.&lt;/p&gt;</comment>
                            <comment id="3368339" author="ian@10gen.com" created="Mon, 31 Aug 2020 18:20:23 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=gregory.wlodarek&quot; class=&quot;user-hover&quot; rel=&quot;gregory.wlodarek&quot;&gt;gregory.wlodarek&lt;/a&gt; after you finish this work can you please talk with Mark so we can figure out what, if anything, we should do with &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-27692&quot; title=&quot;Can diagnostic data capture be stopped later in the shutdown process?&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-27692&quot;&gt;&lt;del&gt;SERVER-27692&lt;/del&gt;&lt;/a&gt;.&lt;/p&gt;</comment>
                            <comment id="3359653" author="haribabu.kommi" created="Wed, 26 Aug 2020 05:30:16 +0000"  >&lt;p&gt;There are two issues that I remember that are happened when WT is shutting down.&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;An increase in memory usage lead to OOM error&lt;/li&gt;
	&lt;li&gt;Shutdown operation is very slow&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;The increase in memory usage that is mentioned in &lt;a href=&quot;https://jira.mongodb.org/browse/WT-6164&quot; title=&quot;Reduce the number of page flags according to the flag variable size&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-6164&quot;&gt;&lt;del&gt;WT-6164&lt;/del&gt;&lt;/a&gt; is related to the handling of clearing the removed keys from the data store led to recursively loading history store pages into the cache. This issue happened during the final checkpoint that happens during WT shutdown.&lt;/p&gt;

&lt;p&gt;The shutdown rollback to stable operation reads many unnecessary pages into cache led to the slowness of the shutdown operation.&lt;/p&gt;

&lt;p&gt;Having FTDC statistics during the shutdown can be identified easily. Performing &lt;tt&gt;rollback to stable&lt;/tt&gt; by MDB itself before calling the connection &lt;tt&gt;close&lt;/tt&gt; can get the metrics related to rollback to stable. Still, we may need to collect all the metrics that are happened during WT shutdown are also required to find out the first problem.&lt;/p&gt;</comment>
                            <comment id="3358865" author="bruce.lucas@10gen.com" created="Tue, 25 Aug 2020 19:29:49 +0000"  >&lt;p&gt;Mostly that makes sense and it&apos;s no loss not to report metrics on subsystems that have shut down, but serverStatus contains some system metrics that would be useful and should be still valid, if it is possible to continue to collect, such as tcmalloc and (less importantly) extra_info.&lt;/p&gt;</comment>
                            <comment id="3358843" author="gregory.wlodarek" created="Tue, 25 Aug 2020 19:17:55 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt;, it should be possible to continue gathering system metrics (external to MongoDB) but not any server status metrics (internal to MongoDB) as the state of the objects that are used to generate the metrics are unknown and may be unsafe to access as shutdown cleans them up.&lt;/p&gt;</comment>
                            <comment id="3358821" author="bruce.lucas@10gen.com" created="Tue, 25 Aug 2020 19:06:31 +0000"  >&lt;p&gt;I think that sounds reasonable (if true).&lt;/p&gt;

&lt;p&gt;If it&apos;s not too much additional work it would also be useful to move ftdc shutdown after wt shutdown (omitting wt metrics) in case there are additional surprisingly expensive operations, or will be in the future. Even though wt metrics would not be available, in such a case there might still be enough information in the remaining metrics, e.g. disk and cpu, to be useful. Also it would prepare for the possibility of improving wt so that its metrics are available during shutdown.&lt;/p&gt;
</comment>
                            <comment id="3358671" author="daniel.gottlieb@10gen.com" created="Tue, 25 Aug 2020 18:01:01 +0000"  >&lt;p&gt;It&apos;s been a while, but I believe &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt; and I discussed this offline, somewhere. IIRC, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt; was mostly interested in having FTDC for WT statistics while WT was shutting down. This specifically became of interest because one of the costs of WT shutdown in 4.2 and earlier was typically bounded by how much data had come into the system since the last checkpoint (or how far the stable timestamp moved since the last checkpoint). But with durable history, the cost is now bounded by how much data exists ahead of the stable timestamp (which is much easier to grow). This is because WT now calls &lt;tt&gt;rollback_to_stable&lt;/tt&gt; inside of &lt;tt&gt;WT_CONNECTION::close&lt;/tt&gt;.&lt;/p&gt;

&lt;p&gt;Assuming that&apos;s the correct motivation/area of interest to target, one compromising solution is that MDB can explicitly call &lt;tt&gt;rollback_to_stable&lt;/tt&gt; on shutdown while FTDC is still running. This would succeed if the following assumptions are true:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;&lt;tt&gt;WT_CONNECTION::close&lt;/tt&gt; would not need to duplicate much, if any of the work accomplished by MDB calling &lt;tt&gt;rollback_to_stable&lt;/tt&gt;.&lt;/li&gt;
	&lt;li&gt;WT produces meaningful metrics while &lt;tt&gt;rollback_to_stable&lt;/tt&gt; is running&lt;/li&gt;
&lt;/ul&gt;
</comment>
                            <comment id="3205898" author="milkie" created="Thu, 11 Jun 2020 19:38:43 +0000"  >&lt;p&gt;For now FTDC will only be able to get metrics outside of WiredTiger, as it is not currently safe to call into WiredTiger to fetch metrics while another thread is in connection-&amp;gt;close().&lt;/p&gt;</comment>
                            <comment id="3130637" author="daniel.gottlieb@10gen.com" created="Wed, 27 May 2020 19:11:37 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt; just to clarify would you expect FTDC to be attempting to get statistics from WT while it&apos;s in &lt;tt&gt;WT_CONNECTION::close&lt;/tt&gt;; it seems like they would be useful, but there&apos;s obviously a safety problem. Or are you only interested in all of the other server metrics?&lt;/p&gt;</comment>
                            <comment id="3086496" author="bruce.lucas@10gen.com" created="Thu, 14 May 2020 17:19:28 +0000"  >&lt;p&gt;Possibly this could be coupled with &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-25042&quot; title=&quot;Start diagnostic data collection as early as possible&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-25042&quot;&gt;&lt;del&gt;SERVER-25042&lt;/del&gt;&lt;/a&gt; (start ftdc as early as possible) as that has also caused diagnostic difficulties.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="1795528">SERVER-58026</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1839821">SERVER-59065</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="300823">SERVER-25042</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="345949">SERVER-27692</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1620674">SERVER-54472</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="1540275">SERVER-52815</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="286920" name="metrics.2020-11-12T15-03-08Z-00000" size="48588" author="gregory.wlodarek@mongodb.com" created="Thu, 12 Nov 2020 15:06:49 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>14.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>6.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="18953"><![CDATA[v4.4]]></customfieldvalue>
    <customfieldvalue key="16775"><![CDATA[v4.2]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 27 May 2020 19:11:37 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        3 years, 8 weeks, 3 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            3 years, 8 weeks, 3 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>daniel.gottlieb@mongodb.com</customfieldvalue>
            <customfieldvalue>milkie@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>gregory.wlodarek@mongodb.com</customfieldvalue>
            <customfieldvalue>haribabu.kommi@mongodb.com</customfieldvalue>
            <customfieldvalue>ian@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hxlb0f:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr4kl3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="3938">Execution Team 2020-06-29</customfieldvalue>
    <customfieldvalue id="3939">Execution Team 2020-07-13</customfieldvalue>
    <customfieldvalue id="3940">Execution Team 2020-07-27</customfieldvalue>
    <customfieldvalue id="4213">Execution Team 2020-09-21</customfieldvalue>
    <customfieldvalue id="4214">Execution Team 2020-10-05</customfieldvalue>
    <customfieldvalue id="4297">Execution Team 2020-11-16</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hxkx9r:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>