<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:10:32 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-46114] Flow-control engages on a single-node replica set</title>
                <link>https://jira.mongodb.org/browse/SERVER-46114</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Flow-control is engaging on single-node replica sets, but it should not.&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2020-02-12T16:33:39.901-0500 W  STORAGE  [FlowControlRefresher] Flow control is engaged and the sustainer point is not moving. Please check the health of all secondaries.&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</description>
                <environment></environment>
        <key id="1145288">SERVER-46114</key>
            <summary>Flow-control engages on a single-node replica set</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13202">Works as Designed</resolution>
                                        <assignee username="maria.vankeulen@mongodb.com">Maria van Keulen</assignee>
                                    <reporter username="geert.bosch@mongodb.com">Geert Bosch</reporter>
                        <labels>
                    </labels>
                <created>Wed, 12 Feb 2020 21:39:30 +0000</created>
                <updated>Fri, 27 Oct 2023 13:52:56 +0000</updated>
                            <resolved>Thu, 16 Apr 2020 20:12:12 +0000</resolved>
                                                                    <component>Replication</component>
                                        <votes>0</votes>
                                    <watches>13</watches>
                                                                                                                <comments>
                            <comment id="3375439" author="mark.callaghan" created="Thu, 3 Sep 2020 19:21:52 +0000"  >&lt;p&gt;Percona shared a bug report with me and this is the cause. See &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-50749&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;SERVER-50749&lt;/a&gt;. The impact is significant. First the load rate drops. Second, some inserts take 200 to 300 seconds when flow control engages.&lt;/p&gt;</comment>
                            <comment id="3260108" author="alexander.gorrod" created="Wed, 1 Jul 2020 11:24:02 +0000"  >&lt;blockquote&gt;&lt;p&gt;From the chart, it would appear that flow control is smoothing out the spikes in throughput over time, at the expense of average throughput.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;It looks like I jumped too soon on this one. I still think it&apos;s surprising that flow control is kicking in on a single node replica set, but it &lt;a href=&quot;https://jira.mongodb.org/browse/WT-6444?focusedCommentId=3260093&amp;amp;page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-3260093&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;isn&apos;t hurting the pytpcc load phase in my testing&lt;/a&gt;, so that isn&apos;t a reason to resurrect this ticket.&lt;/p&gt;</comment>
                            <comment id="3230113" author="bruce.lucas@10gen.com" created="Tue, 30 Jun 2020 14:37:02 +0000"  >&lt;p&gt;For the purposes of doing that comparison it might be useful to disable flow control on both 4.2 and 4.4 in order to remove that complication and focus on measuring the relative storage engine performance.&lt;/p&gt;</comment>
                            <comment id="3230093" author="milkie" created="Tue, 30 Jun 2020 14:31:26 +0000"  >&lt;p&gt;We could consider it, but my understanding of &lt;a href=&quot;https://jira.mongodb.org/browse/WT-6444&quot; title=&quot;Abort a transaction if it is force evicting and oldest&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-6444&quot;&gt;&lt;del&gt;WT-6444&lt;/del&gt;&lt;/a&gt; is that it is investigating the performance difference between 4.2 and 4.4, and for both of those versions the flow control logic is the same.&lt;/p&gt;

&lt;p&gt;From the chart, it would appear that flow control is smoothing out the spikes in throughput over time, at the expense of average throughput. &#160;Wouldn&apos;t a chart look similar for a multi-node replica set? &#160;I am not sure if users would prefer more spiky performance or more stable performance, and also not sure if that preference is different for one-node replica sets.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;</comment>
                            <comment id="3229768" author="alexander.gorrod" created="Tue, 30 Jun 2020 11:49:59 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=milkie&quot; class=&quot;user-hover&quot; rel=&quot;milkie&quot;&gt;milkie&lt;/a&gt; and &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=michael.gargiulo&quot; class=&quot;user-hover&quot; rel=&quot;michael.gargiulo&quot;&gt;michael.gargiulo&lt;/a&gt; I&apos;ve been running the pytpcc load workload described in &lt;a href=&quot;https://jira.mongodb.org/browse/WT-6444&quot; title=&quot;Abort a transaction if it is force evicting and oldest&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-6444&quot;&gt;&lt;del&gt;WT-6444&lt;/del&gt;&lt;/a&gt;, and trying to improve the performance of WiredTiger with that workload. The workload is running a 1-node replica set, and I&apos;m seeing that flow control is kicking in and causing the average throughput to reduce from 35k to 27k operations/second. See the picture below which shows a lot of threads waiting on flow control. The throughput is also less stable, but we have other changes in testing which should make the throughput more stable, and achieve an even higher throughput.&lt;/p&gt;

&lt;p&gt;Would you reconsider whether flow control should be disabled in 1-node replica set configurations?&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/267917/267917_SERVER-46114.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;</comment>
                            <comment id="3043371" author="maria.vankeulen" created="Thu, 16 Apr 2020 20:49:17 +0000"  >&lt;p&gt;In case anybody is curious, I also did a run of this workload with Flow Control disabled entirely, with similar write latency spikes and durable/lastCommitted lag:&lt;br/&gt;
 &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/256340/256340_No+Flow+Control.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt; &lt;/p&gt;

&lt;p&gt;Here&apos;s a side-by-side comparison, including throughput:&lt;br/&gt;
 &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/256342/256342_Comparison.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt; &lt;/p&gt;</comment>
                            <comment id="3043294" author="maria.vankeulen" created="Thu, 16 Apr 2020 20:10:30 +0000"  >&lt;p&gt;It looks like the lastCommitted lag coincides with the lastDurable lag for this workload:&lt;br/&gt;
 &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/256333/256333_lastDurable+lag+comparison.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;The lastDurable OpTime is &lt;a href=&quot;https://github.com/mongodb/mongo/blob/r4.2.5/src/mongo/db/repl/topology_coordinator.cpp#L2422&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;used&lt;/a&gt; to determine the lastCommitted OpTime &lt;a href=&quot;https://docs.mongodb.com/manual/reference/replica-configuration/#rsconf.writeConcernMajorityJournalDefault&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;by default&lt;/a&gt;. So, any lag between lastApplied and lastDurable can influence lastCommitted lag.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=daniel.gottlieb&quot; class=&quot;user-hover&quot; rel=&quot;daniel.gottlieb&quot;&gt;daniel.gottlieb&lt;/a&gt; also pointed out that an overwhelmed system could starve writers long enough for Flow Control to kick in.&lt;/p&gt;

&lt;p&gt;There are spikes in write latency throughout this workload, even when Flow Control isn&apos;t engaged, suggesting strain on the mongod. Here&apos;s an example of such spikes in a section where Flow Control was not throttling any writes:&lt;br/&gt;
 &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/256334/256334_Flow+Control+disengaged.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;I think the lag in lastDurable is due to the strain on the system, and we can conclude that the Flow Control lag measure can be overstated when the node itself is under strain. There is an existing ticket, &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-45880&quot; title=&quot;Flow Control lag detection mechanism can overstate lag if there are oplog holes&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-45880&quot;&gt;&lt;del&gt;SERVER-45880&lt;/del&gt;&lt;/a&gt;, to reconsider whether lastApplied versus lastCommitted is the best measure of lag. Other than that, I don&apos;t think there&apos;s any further work that needs to be done for &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-46114&quot; title=&quot;Flow-control engages on a single-node replica set&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-46114&quot;&gt;&lt;del&gt;SERVER-46114&lt;/del&gt;&lt;/a&gt;.&lt;/p&gt;</comment>
                            <comment id="3041386" author="maria.vankeulen" created="Wed, 15 Apr 2020 20:09:32 +0000"  >&lt;p&gt;I was able to reproduce this issue locally. I looked at the FTDC data, and Flow Control&apos;s lag detection does report &quot;lag&quot;.&lt;br/&gt;
Flow Control&apos;s &lt;a href=&quot;https://github.com/mongodb/mongo/blob/r4.2.5/src/mongo/db/storage/flow_control.cpp#L338-L341&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;lag check&lt;/a&gt; takes a few factors into consideration. Most notably, it considers the difference between the &lt;tt&gt;lastCommitted&lt;/tt&gt; wall clock time and the &lt;tt&gt;lastApplied&lt;/tt&gt; wall clock time measured on the primary. As of &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-42917&quot; title=&quot;Flow Control should disregard faulty wall clock time readings during lag calculation&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-42917&quot;&gt;&lt;del&gt;SERVER-42917&lt;/del&gt;&lt;/a&gt;, it is not possible for this lag check to be thrown off by a &lt;tt&gt;lastCommitted&lt;/tt&gt; wall clock time that happens to be greater than &lt;tt&gt;lastApplied&lt;/tt&gt; wall clock time (see comment thread in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-42906&quot; title=&quot;lastCommitted wall clock time can be greater than lastApplied wall clock time on the primary&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-42906&quot;&gt;&lt;del&gt;SERVER-42906&lt;/del&gt;&lt;/a&gt; for a description of how this can happen).&lt;br/&gt;
However, I found in my repro that it is possible for the &lt;tt&gt;lastCommitted&lt;/tt&gt; wall clock time (and associated OpTime) to be &lt;b&gt;behind&lt;/b&gt; the &lt;tt&gt;lastApplied&lt;/tt&gt; wall clock time/OpTime, even in a single-node replica set. As a result, Flow Control will report a &quot;lag&quot;, and throttle the primary.&lt;br/&gt;
I&apos;m not sure how this can be the case, so I&apos;ll continue to investigate. I&apos;ll keep this ticket posted with any findings.&lt;br/&gt;
 &lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/256166/256166_single+node+rs+lag.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt; &lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="1122533">SERVER-45880</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="256342" name="Comparison.png" size="147339" author="maria.vankeulen@mongodb.com" created="Thu, 16 Apr 2020 21:07:01 +0000"/>
                            <attachment id="256334" name="Flow Control disengaged.png" size="51995" author="maria.vankeulen@mongodb.com" created="Thu, 16 Apr 2020 20:07:59 +0000"/>
                            <attachment id="256340" name="No Flow Control.png" size="49647" author="maria.vankeulen@mongodb.com" created="Thu, 16 Apr 2020 20:48:42 +0000"/>
                            <attachment id="267917" name="SERVER-46114.png" size="56603" author="alexander.gorrod@mongodb.com" created="Tue, 30 Jun 2020 11:49:04 +0000"/>
                            <attachment id="256333" name="lastDurable lag comparison.png" size="64492" author="maria.vankeulen@mongodb.com" created="Thu, 16 Apr 2020 20:07:54 +0000"/>
                            <attachment id="256166" name="single node rs lag.png" size="40540" author="maria.vankeulen@mongodb.com" created="Wed, 15 Apr 2020 20:08:28 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>8.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 24 Feb 2020 21:06:48 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        3 years, 22 weeks, 6 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            3 years, 22 weeks, 6 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>alexander.gorrod@mongodb.com</customfieldvalue>
            <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>milkie@mongodb.com</customfieldvalue>
            <customfieldvalue>geert.bosch@mongodb.com</customfieldvalue>
            <customfieldvalue>maria.vankeulen@mongodb.com</customfieldvalue>
            <customfieldvalue>mark.callaghan@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hwq83z:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hwvb2f:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="3634">Execution Team 2020-04-20</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10750" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Steps To Reproduce</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>&lt;p&gt;The log file includes messages like:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2020-02-12T16:33:39.901-0500 W  STORAGE  [FlowControlRefresher] Flow control is engaged and the sustainer point is not moving. Please check the health of all secondaries.&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hwpudb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>