<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:50:37 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-19334] TTL index deletions cannot always keep up with insertions with WiredTiger</title>
                <link>https://jira.mongodb.org/browse/SERVER-19334</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt; Configuration:&lt;/p&gt;

&lt;p&gt;db version v3.1.6-pre-&lt;br/&gt;
git version: 21c4ce42c34008a966e66214b1dd83cdc066bf1c&lt;/p&gt;

&lt;p&gt;Environment &lt;/p&gt;
&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;Windows&lt;/li&gt;
	&lt;li&gt;SA or Replica set&lt;/li&gt;
	&lt;li&gt;Storage engine WT&lt;/li&gt;
	&lt;li&gt;TTL configured to 20 sec&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Workload:&lt;/p&gt;

&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;Hammer.mongo insert only profile w/ TTL = 20 sec (configurable variable in simple_insert.sh) i&apos;m using it to simulate cache workload&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Issues:&lt;/p&gt;

&lt;p&gt;MongoDb reported that document has been deleted w/ &lt;br/&gt;
 db.serverStatus().metrics.ttl&lt;br/&gt;
But all document remains and DB size is growing like insert only no TTL , and you can see them using db.c.find()&lt;/p&gt;

&lt;p&gt;TTL working well on the same workload , same build w/ MMAP storage engine &lt;/p&gt;

&lt;h6&gt;&lt;a name=&quot;Additionalinformation&quot;&gt;&lt;/a&gt;Additional information&lt;/h6&gt;
&lt;p&gt;Please see &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-19688&quot; title=&quot;TTL Index is having more Latency&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-19688&quot;&gt;&lt;del&gt;SERVER-19688&lt;/del&gt;&lt;/a&gt; as well.&lt;/p&gt;</description>
                <environment></environment>
        <key id="215708">SERVER-19334</key>
            <summary>TTL index deletions cannot always keep up with insertions with WiredTiger</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="4">Incomplete</resolution>
                                        <assignee username="backlog-server-execution">Backlog - Storage Execution Team</assignee>
                                    <reporter username="eitan.klein">Eitan Klein</reporter>
                        <labels>
                            <label>32qa</label>
                    </labels>
                <created>Wed, 8 Jul 2015 19:05:19 +0000</created>
                <updated>Tue, 6 Dec 2022 04:48:34 +0000</updated>
                            <resolved>Thu, 2 Apr 2020 20:09:24 +0000</resolved>
                                                                    <component>Performance</component>
                    <component>Storage</component>
                    <component>WiredTiger</component>
                                        <votes>1</votes>
                                    <watches>26</watches>
                                                                                                                <comments>
                            <comment id="1063370" author="xgen-internal-githook" created="Fri, 16 Oct 2015 21:53:47 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;GeertBosch&apos;, u&apos;name&apos;: u&apos;Geert Bosch&apos;, u&apos;email&apos;: u&apos;geert@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-19334&quot; title=&quot;TTL index deletions cannot always keep up with insertions with WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-19334&quot;&gt;&lt;del&gt;SERVER-19334&lt;/del&gt;&lt;/a&gt;: Refactor handling of status in delete stage&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/ec9317104da7816cabb41cf603c084099d768df0&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/ec9317104da7816cabb41cf603c084099d768df0&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="994750" author="dan@10gen.com" created="Mon, 3 Aug 2015 03:18:24 +0000"  >&lt;p&gt;re-opening this issue to pursue approach of batching deletes &lt;/p&gt;</comment>
                            <comment id="964762" author="geert.bosch" created="Mon, 13 Jul 2015 18:19:34 +0000"  >&lt;p&gt;The issue is that restoreState resets the timer, and we call this function in the delete loop of doTTLForIndex:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;(gdb) bt&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#0  mongo::ElapsedTracker::resetLastTime (this=0x12b07a58) at src/mongo/util/elapsed_tracker.cpp:62&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#1  0x0000000000c7ac5e in mongo::PlanYieldPolicy::resetTimer (this=&amp;lt;optimized out&amp;gt;) at src/mongo/db/query/plan_yield_policy.cpp:57&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#2  0x0000000000c75024 in mongo::PlanExecutor::restoreStateWithoutRetrying (this=this@entry=0x119d88f0, opCtx=opCtx@entry=0x7f371147d870)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    at src/mongo/db/query/plan_executor.cpp:290&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#3  0x0000000000c75091 in mongo::PlanExecutor::restoreState (this=0x119d88f0, opCtx=opCtx@entry=0x7f371147d870) at src/mongo/db/query/plan_executor.cpp:272&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#4  0x0000000000eb8b61 in mongo::TTLMonitor::doTTLForIndex (this=this@entry=0x3341ba0, txn=txn@entry=0x7f371147d870, dbName=..., idx=...) at src/mongo/db/ttl.cpp:307&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#5  0x0000000000eb9b92 in mongo::TTLMonitor::doTTLPass (this=this@entry=0x3341ba0) at src/mongo/db/ttl.cpp:145&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#6  0x0000000000eba1f8 in mongo::TTLMonitor::run (this=0x3341ba0) at src/mongo/db/ttl.cpp:113&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#7  0x000000000105da07 in mongo::BackgroundJob::jobBody (this=0x3341ba0) at src/mongo/util/background.cpp:152&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#8  0x00007f3716b15b60 in std::(anonymous namespace)::execute_native_thread_routine (__p=&amp;lt;optimized out&amp;gt;) at ../../../../../libstdc++-v3/src/c++11/thread.cc:84&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#9  0x00007f3716328182 in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;#10 0x00007f3716052fbd in clone () from /lib/x86_64-linux-gnu/libc.so.6&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;As a result the TTL monitor never yields.&lt;/p&gt;</comment>
                            <comment id="964612" author="dan@10gen.com" created="Mon, 13 Jul 2015 16:30:30 +0000"  >&lt;p&gt;Broke out the separate issue found while investigating this one here:  &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-19374&quot; title=&quot;TTL Monitor does not yield while deleting expired documents&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-19374&quot;&gt;&lt;del&gt;SERVER-19374&lt;/del&gt;&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="963745" author="dan@10gen.com" created="Fri, 10 Jul 2015 22:54:11 +0000"  >&lt;p&gt;This did expose a possible issue with the TTL monitor though.  When the TTL monitor is running, I found that listCollections would block.  Easy to repro:&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;Create a ttl index&lt;/li&gt;
	&lt;li&gt;Start an insert job into that collection (benchrun will work)&lt;/li&gt;
	&lt;li&gt;When the TTL monitor job is running, try running db.getCollectionNames().  It will block til the monitor is done deleting docs.&lt;/li&gt;
&lt;/ul&gt;
</comment>
                            <comment id="963739" author="dan@10gen.com" created="Fri, 10 Jul 2015 22:29:57 +0000"  >&lt;p&gt;Depending exactly on the parameters used in the test, it is very easy to outstrip the ability of mongod to delete as much as what is being inserted.  The TTL monitor runs 1/minute and is single threaded, deleting each document 1 at a time.  This can be exacerbated with a small cache, which will force the deleted documents to be pulled off disk.  With MMAP, I believe the insertion rate is naturally throttled by the deletions because of the collection level lock.&lt;/p&gt;

&lt;p&gt;This explains why the data would take a long time to be deleted.  The disk space then would only be reclaimed gradually by new data, but without an explicit compact, I don&apos;t think it would be returned to the system, and thus would stay at the high water mark it reached during the insertion phase.&lt;/p&gt;

&lt;p&gt;I don&apos;t think there is a ton we can do right now to make this much better, other than possibly parallelizing the deletes in some manner.&lt;/p&gt;</comment>
                            <comment id="961555" author="samk" created="Wed, 8 Jul 2015 21:04:37 +0000"  >&lt;p&gt;The TTL process only deletes documents every 60 seconds, and the configurability of the interval that the TTL deletion operation is not documented (noted as &quot;for testing only&quot;.) &lt;a href=&quot;https://github.com/mongodb/mongo/blob/master/src/mongo/db/ttl.cpp#L78&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/blob/master/src/mongo/db/ttl.cpp#L78&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;The &lt;a href=&quot;http://docs.mongodb.org/manual/core/index-ttl/#timing-of-the-delete-operation&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;TTL documentation&lt;/a&gt; states that expired documents may continue to exist in the database for some time after they&apos;ve reached the timeout. Thus this is expected behavior.&lt;/p&gt;

&lt;p&gt;Cheers,&lt;br/&gt;
sam&lt;/p&gt;</comment>
                            <comment id="961527" author="eitan.klein" created="Wed, 8 Jul 2015 20:35:02 +0000"  >&lt;p&gt;Easy repro script&lt;/p&gt;

&lt;p&gt;db.foo.createIndex(&lt;/p&gt;
{date: 1}
&lt;p&gt;, &lt;/p&gt;
{expireAfterSeconds: 20}
&lt;p&gt;)&lt;br/&gt;
for (var i=0; i&amp;lt;100;i++) { db.foo.insert(&lt;/p&gt;
{x: i, date: new Date()}
&lt;p&gt;) }&lt;br/&gt;
Wait 20 sec&lt;br/&gt;
db.foo.find() should return an empty set&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                                                <inwardlinks description="is duplicated by">
                                        <issuelink>
            <issuekey id="223553">SERVER-19688</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                                        </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="81060" name="FindOne and Index.txt" size="2475" author="eitan.klein" created="Wed, 8 Jul 2015 20:25:36 +0000"/>
                            <attachment id="81059" name="serverstatus.txt" size="13427" author="eitan.klein" created="Wed, 8 Jul 2015 20:25:36 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>8.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_12751" key="com.atlassian.jira.plugin.system.customfieldtypes:multiselect">
                        <customfieldname>Assigned Teams</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="25136"><![CDATA[Storage Execution]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 8 Jul 2015 21:04:37 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        8 years, 17 weeks, 5 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>alexander.golin@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            8 years, 17 weeks, 5 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>backlog-server-execution</customfieldvalue>
            <customfieldvalue>dan@mongodb.com</customfieldvalue>
            <customfieldvalue>eitan.klein</customfieldvalue>
            <customfieldvalue>geert.bosch@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>sam.kleinman</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrl127:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr5iuf:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="878">Integration F (02/01/16)</customfieldvalue>
    <customfieldvalue id="912">Integration 10 (02/22/16)</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hsbzef:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>