<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:35:39 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-34129] Don&apos;t hold database or collection locks during oplog truncation</title>
                <link>https://jira.mongodb.org/browse/SERVER-34129</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Some customers are sensitive to the short stalls that can be caused by oplog truncation, which can be a moderately heavy operation. We could mitigate this issue by providing a parameter to tune the amount of oplog data truncated in a single operation.&lt;/p&gt;</description>
                <environment></environment>
        <key id="516626">SERVER-34129</key>
            <summary>Don&apos;t hold database or collection locks during oplog truncation</summary>
                <type id="4" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14710&amp;avatarType=issuetype">Improvement</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="xiangyu.yao@mongodb.com">Xiangyu Yao</assignee>
                                    <reporter username="bruce.lucas@mongodb.com">Bruce Lucas</reporter>
                        <labels>
                            <label>QFB</label>
                            <label>nyc</label>
                    </labels>
                <created>Mon, 26 Mar 2018 14:07:34 +0000</created>
                <updated>Sun, 29 Oct 2023 22:33:21 +0000</updated>
                            <resolved>Thu, 5 Jul 2018 15:19:04 +0000</resolved>
                                                    <fixVersion>3.6.7</fixVersion>
                    <fixVersion>4.0.1</fixVersion>
                    <fixVersion>4.1.1</fixVersion>
                                    <component>Storage</component>
                    <component>WiredTiger</component>
                                        <votes>0</votes>
                                    <watches>18</watches>
                                                                                                                <comments>
                            <comment id="1952817" author="xgen-internal-githook" created="Fri, 20 Jul 2018 21:37:56 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;username&apos;: &apos;xy24&apos;, &apos;name&apos;: &apos;Xiangyu Yao&apos;, &apos;email&apos;: &apos;xiangyu.yao@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-34129&quot; title=&quot;Don&amp;#39;t hold database or collection locks during oplog truncation&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-34129&quot;&gt;&lt;del&gt;SERVER-34129&lt;/del&gt;&lt;/a&gt; Don&apos;t hold database or collection locks during oplog truncation&lt;/p&gt;

&lt;p&gt;(cherry picked from commit 9008da291dd91dae2ecf0529e1fa8a31329389c5)&lt;br/&gt;
Branch: v3.6&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/64f52ceeabf9a39603574e41b527c16239e10df3&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/64f52ceeabf9a39603574e41b527c16239e10df3&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1941844" author="xgen-internal-githook" created="Mon, 9 Jul 2018 14:27:33 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;username&apos;: &apos;xy24&apos;, &apos;name&apos;: &apos;Xiangyu Yao&apos;, &apos;email&apos;: &apos;xiangyu.yao@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-34129&quot; title=&quot;Don&amp;#39;t hold database or collection locks during oplog truncation&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-34129&quot;&gt;&lt;del&gt;SERVER-34129&lt;/del&gt;&lt;/a&gt; Don&apos;t hold database or collection locks during oplog truncation&lt;/p&gt;

&lt;p&gt;(cherry picked from commit 9008da291dd91dae2ecf0529e1fa8a31329389c5)&lt;br/&gt;
Branch: v4.0&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/c6d658f7e78a792cd50fc47ce91a2ee3ba98580a&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/c6d658f7e78a792cd50fc47ce91a2ee3ba98580a&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1939390" author="xgen-internal-githook" created="Thu, 5 Jul 2018 15:14:39 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;username&apos;: &apos;xy24&apos;, &apos;name&apos;: &apos;Xiangyu Yao&apos;, &apos;email&apos;: &apos;xiangyu.yao@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-34129&quot; title=&quot;Don&amp;#39;t hold database or collection locks during oplog truncation&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-34129&quot;&gt;&lt;del&gt;SERVER-34129&lt;/del&gt;&lt;/a&gt; Don&apos;t hold database or collection locks during oplog truncation&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/9008da291dd91dae2ecf0529e1fa8a31329389c5&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/9008da291dd91dae2ecf0529e1fa8a31329389c5&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1929133" author="xiangyu.yao" created="Fri, 22 Jun 2018 17:59:54 +0000"  >&lt;p&gt;Potential blocking commands/operations:&lt;/p&gt;

&lt;p&gt;AppendOplogNote command takes a global X lock in&#160;3.4, but IX lock in&#160;3.6 and 4.0.&lt;/p&gt;

&lt;p&gt;DropDatabase takes global X locks in 3.4, 3.6 and 4.0.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;</comment>
                            <comment id="1928135" author="xiangyu.yao" created="Thu, 21 Jun 2018 20:01:39 +0000"  >&lt;p&gt;It&apos;s hard to remove all the locks of oplog collection and treat it specially. The main issue is from restarting the catalog by rollback, when we need to hold a reference to the oplog collection on closeCatalog and later restore it. It requires new logic specifically for oplog collection. Also, the change cannot be easily backported to v3.4 and v3.6 because of different rollback logic. If the main issue here is that oplog truncation blocks oplog application on the secondary, we should opt out of the PBWM at oplog truncation.&lt;/p&gt;

&lt;p&gt;Furthermore, if we don&apos;t want the truncation to block listCollections which takes a database S lock, we could let oplog truncation take a Global IX (which prevents restartCatalog) and don&apos;t take database/collection IX locks.&lt;/p&gt;</comment>
                            <comment id="1917260" author="bruce.lucas@10gen.com" created="Mon, 11 Jun 2018 17:38:00 +0000"  >&lt;p&gt;OK, that sounds good. So maybe we should change the headline again to &quot;Don&apos;t hold any locks during oplog truncation&quot;?&lt;/p&gt;</comment>
                            <comment id="1917251" author="milkie" created="Mon, 11 Jun 2018 17:36:04 +0000"  >&lt;p&gt;An intent lock is actually not required here.  We&apos;ve been playing around with the idea that the oplog can be treated specially w.r.t. locking since it already has special properties applied to it (such as it can never be dropped), which allows us to relax the locking model a bit.&lt;/p&gt;</comment>
                            <comment id="1917214" author="bruce.lucas@10gen.com" created="Mon, 11 Jun 2018 17:19:39 +0000"  >&lt;p&gt;If I understand correctly that won&apos;t help with the original issue that this ticket was opened for which was higher latency on secondaries during oplog truncation on 3.4 and 3.6, will it? At least an intent lock will still be required which will block the oplog application global lock which will block all other reads. (That particular problem won&apos;t exist in 4.0 though.)&lt;/p&gt;</comment>
                            <comment id="1917015" author="milkie" created="Mon, 11 Jun 2018 15:30:40 +0000"  >&lt;p&gt;We&apos;ve decided the best way forward here is to change the coding logic such that no database locks are held while oplog truncation is occurring, to prevent it from affecting other operations if it takes a long time.&lt;/p&gt;</comment>
                            <comment id="1909283" author="alexander.gorrod" created="Mon, 4 Jun 2018 00:51:31 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=michael.cahill&quot; class=&quot;user-hover&quot; rel=&quot;michael.cahill&quot;&gt;michael.cahill&lt;/a&gt; The alternative WiredTiger API to limit how much data truncate will remove feels like it will introduce ongoing maintenance issues, as such I prefer your suggestion to:&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;It would also relatively easy for the reclaimOplog to extrapolate from the total number of entries in the oplog and estimate how many will be in the 1% being truncated, then interpolate to truncate in bite-sized pieces. This won&apos;t deal with very bursty workloads, but may be good enough for practical purposes.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Another potential avenue for alleviating the symptom might be to not hold a database lock while doing the truncate?&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=milkie&quot; class=&quot;user-hover&quot; rel=&quot;milkie&quot;&gt;milkie&lt;/a&gt; or &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=geert.bosch&quot; class=&quot;user-hover&quot; rel=&quot;geert.bosch&quot;&gt;geert.bosch&lt;/a&gt; could you pick this one up?&lt;/p&gt;</comment>
                            <comment id="1904453" author="bruce.lucas@10gen.com" created="Wed, 30 May 2018 12:37:08 +0000"  >&lt;p&gt;I suspect that generally customers who care about the small additional latencies we&apos;re talking about here probably don&apos;t have very bursty workloads, because that would make latencies unpredictable.&lt;/p&gt;</comment>
                            <comment id="1904117" author="michael.cahill" created="Wed, 30 May 2018 02:48:42 +0000"  >&lt;blockquote&gt;&lt;p&gt;The caller won&apos;t know what data was deleted after the call has completed.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;True, but we could return a result like &lt;tt&gt;WT_RESTART&lt;/tt&gt; / &lt;tt&gt;EAGAIN&lt;/tt&gt; to indicate that the application should keep calling truncate (with the same key range) until it is successful.&lt;/p&gt;

&lt;p&gt;It would also relatively easy for the &lt;tt&gt;reclaimOplog&lt;/tt&gt; to extrapolate from the total number of entries in the oplog and estimate how many will be in the 1% being truncated, then interpolate to truncate in bite-sized pieces.  This won&apos;t deal with very bursty workloads, but may be good enough for practical purposes.&lt;/p&gt;</comment>
                            <comment id="1886541" author="alexander.gorrod" created="Wed, 9 May 2018 01:36:44 +0000"  >&lt;blockquote&gt;&lt;p&gt;We could consider driving down into WT&apos;s truncate operation a maximum number of pages or maximum time to make the MongoDB code more straightforward.&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;The semantic feels quite strange if we try to push this problem down inside WiredTiger - the truncate API would need to be &quot;delete this data, or as much as you can within X milliseconds&quot;. The caller won&apos;t know what data was deleted after the call has completed.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=milkie&quot; class=&quot;user-hover&quot; rel=&quot;milkie&quot;&gt;milkie&lt;/a&gt; Would it be practical to instead track more information inside an individual oplog stone via &lt;a href=&quot;https://github.com/mongodb/mongo/blob/master/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp#L108&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;WiredTigerRecordStore::OplogStones::InsertChange&lt;/a&gt; to track records as the stone is populated i.e: one every 1000 records (or some number), then truncate for a stone could be broken into several parts. The code in WiredTigerRecordStore::reclaimOplog would need to become more sophisticated in terms of continuing to work in the face of partially completing  a reclaim operation, but I think that&apos;s the case no matter which approach is taken.&lt;/p&gt;</comment>
                            <comment id="1864913" author="milkie" created="Mon, 16 Apr 2018 12:56:34 +0000"  >&lt;p&gt;I wonder if we could put a ceiling on the number of documents that are encapsulated between each Stone; this would prevent really-large oplog sizes from truncating the oplog too coarsely.  It would possibly increase startup time, though, while the Stones are initialized.&lt;/p&gt;</comment>
                            <comment id="1864802" author="michael.cahill" created="Mon, 16 Apr 2018 08:28:08 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=milkie&quot; class=&quot;user-hover&quot; rel=&quot;milkie&quot;&gt;milkie&lt;/a&gt;, I don&apos;t think truncate is doing significant amount of I/O relative to the size of the truncation.  The issue as &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt; pointed out in the linked ticket is &quot;the truncation is done as a single unyielding operation, so it holds an intent lock... This stalls replication waiting for global W lock which in turn stalls everything else while the truncate runs.&quot;&lt;/p&gt;

&lt;p&gt;One option would be to keep the current OplogStones logic that tracks markers every ~1% of the oplog, but perform a series of truncate operations (yielding in between) to free each range.  We could consider driving down into WT&apos;s truncate operation a maximum number of pages or maximum time to make the MongoDB code more straightforward.&lt;/p&gt;</comment>
                            <comment id="1863545" author="milkie" created="Fri, 13 Apr 2018 14:18:06 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=michael.cahill&quot; class=&quot;user-hover&quot; rel=&quot;michael.cahill&quot;&gt;michael.cahill&lt;/a&gt; might this be a result of truncate using more I/O than expected?  Are there changes we can do to improve this in WiredTiger itself?&lt;/p&gt;</comment>
                            <comment id="1862722" author="bruce.lucas@10gen.com" created="Thu, 12 Apr 2018 17:26:40 +0000"  >&lt;p&gt;This may not require a parameter. What is really needed is to ensure that the oplog truncation doesn&apos;t take too long while holding locks. Truncating 1% of the oplog in a single WT truncate operation may violate this. If we could arrange our truncations in smaller batches so they each complete in a short time that would be sufficient.&lt;/p&gt;</comment>
                            <comment id="1850061" author="ian@10gen.com" created="Fri, 30 Mar 2018 14:12:22 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=asya&quot; class=&quot;user-hover&quot; rel=&quot;asya&quot;&gt;asya&lt;/a&gt; to talk with &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt; about what the best solution is for the underlying problem.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="854000">SERVER-42175</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>18.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>3.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="15640"><![CDATA[v4.0]]></customfieldvalue>
    <customfieldvalue key="15141"><![CDATA[v3.6]]></customfieldvalue>
    <customfieldvalue key="14340"><![CDATA[v3.4]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[500A000000a9JrPIAU, 500A000000atsQ1IAI, 500A000000bQueQIAS, 5002K00000dazYfQAI]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 30 Mar 2018 14:12:22 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        5 years, 29 weeks, 5 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            5 years, 29 weeks, 5 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>alexander.gorrod@mongodb.com</customfieldvalue>
            <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>milkie@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>ian@mongodb.com</customfieldvalue>
            <customfieldvalue>michael.cahill@mongodb.com</customfieldvalue>
            <customfieldvalue>xiangyu.yao@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|httk6f:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|htr48n:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="2329">Storage NYC 2018-06-18</customfieldvalue>
    <customfieldvalue id="2391">Storage NYC 2018-07-02</customfieldvalue>
    <customfieldvalue id="2393">Storage NYC 2018-07-16</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htt6e7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>