<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:49:27 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-18983] Process oplog inserts, and applying, on the secondary in parallel</title>
                <link>https://jira.mongodb.org/browse/SERVER-18983</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Do not wait for the apply of the oplog entries before recording them in the oplog. This means both of these operation can be done concurrently instead of serially now that we record the boundaries of the batch and recover correctly but removing the oplog entries record during failures.&lt;/p&gt;

&lt;p&gt;&lt;b&gt;old description&lt;/b&gt;&lt;br/&gt;
Inserts into the oplog on the primary are done in parallel by each connection thread, whereas they are done on the secondary serially by the sync thread. This means that the oplog inserts are considerably slower on the secondary, which can create replication lag. See &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-18908?focusedCommentId=940680&amp;amp;page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-940680&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;this comment&lt;/a&gt; for more information.&lt;/p&gt;</description>
                <environment></environment>
        <key id="210754">SERVER-18983</key>
            <summary>Process oplog inserts, and applying, on the secondary in parallel</summary>
                <type id="3" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14718&amp;avatarType=issuetype">Task</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="backlog-server-tuning">Backlog - Tuning Team</assignee>
                                    <reporter username="bruce.lucas@mongodb.com">Bruce Lucas</reporter>
                        <labels>
                            <label>mms-s</label>
                    </labels>
                <created>Mon, 15 Jun 2015 19:52:42 +0000</created>
                <updated>Mon, 2 Oct 2017 18:04:41 +0000</updated>
                            <resolved>Mon, 2 Oct 2017 18:04:41 +0000</resolved>
                                                                    <component>Replication</component>
                    <component>Storage</component>
                                        <votes>4</votes>
                                    <watches>29</watches>
                                                                                                                <comments>
                            <comment id="1686818" author="redbeard0531" created="Mon, 2 Oct 2017 18:04:41 +0000"  >&lt;p&gt;This work was done a while ago under &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-24242&quot; title=&quot;Improve replication performance&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-24242&quot;&gt;&lt;del&gt;SERVER-24242&lt;/del&gt;&lt;/a&gt; and &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-7200&quot; title=&quot;use oplog as op buffer on secondaries&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-7200&quot;&gt;&lt;del&gt;SERVER-7200&lt;/del&gt;&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1058086" author="xgen-internal-githook" created="Mon, 12 Oct 2015 16:27:35 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;scotthernandez&apos;, u&apos;name&apos;: u&apos;Scott Hernandez&apos;, u&apos;email&apos;: u&apos;scotthernandez@gmail.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-18983&quot; title=&quot;Process oplog inserts, and applying, on the secondary in parallel&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-18983&quot;&gt;&lt;del&gt;SERVER-18983&lt;/del&gt;&lt;/a&gt;: Apply oplog and record in oplog concurrently&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/cc1f48bce42728f3af21e8c6d3a9766f3675ac8a&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/cc1f48bce42728f3af21e8c6d3a9766f3675ac8a&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1049674" author="xgen-internal-githook" created="Fri, 2 Oct 2015 11:17:51 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;scotthernandez&apos;, u&apos;name&apos;: u&apos;Scott Hernandez&apos;, u&apos;email&apos;: u&apos;scotthernandez@gmail.com&apos;}
&lt;p&gt;Message: Revert &quot;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-18983&quot; title=&quot;Process oplog inserts, and applying, on the secondary in parallel&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-18983&quot;&gt;&lt;del&gt;SERVER-18983&lt;/del&gt;&lt;/a&gt;: Apply oplog and record in oplog concurrently&quot;&lt;/p&gt;

&lt;p&gt;This reverts commit 3937e8a5a855aebc4c8e16206fd69c863f567e15.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/f25e8acf1a160bbfa39035888bb026049b10ae22&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/f25e8acf1a160bbfa39035888bb026049b10ae22&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1048756" author="xgen-internal-githook" created="Thu, 1 Oct 2015 15:40:36 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;scotthernandez&apos;, u&apos;name&apos;: u&apos;Scott Hernandez&apos;, u&apos;email&apos;: u&apos;scotthernandez@gmail.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-18983&quot; title=&quot;Process oplog inserts, and applying, on the secondary in parallel&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-18983&quot;&gt;&lt;del&gt;SERVER-18983&lt;/del&gt;&lt;/a&gt;: just check for timing field, not validation of value&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/ca4481c3269768e196ad8d7594c0d84dfe4f4593&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/ca4481c3269768e196ad8d7594c0d84dfe4f4593&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1048647" author="martin.bligh" created="Thu, 1 Oct 2015 14:40:21 +0000"  >&lt;p&gt;Re-opening this ticket and assigning back to me, because we still have to investigate Bruce&apos;s original point of doing this in parallel&lt;/p&gt;

&lt;p&gt;Most of the work Scott has been doing is a pre-req for this, as it liberates us to do oplog vs collections in much less strict order.&lt;/p&gt;</comment>
                            <comment id="1048549" author="xgen-internal-githook" created="Thu, 1 Oct 2015 13:46:55 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;scotthernandez&apos;, u&apos;name&apos;: u&apos;Scott Hernandez&apos;, u&apos;email&apos;: u&apos;scotthernandez@gmail.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-18983&quot; title=&quot;Process oplog inserts, and applying, on the secondary in parallel&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-18983&quot;&gt;&lt;del&gt;SERVER-18983&lt;/del&gt;&lt;/a&gt;: Apply oplog and record in oplog concurrently&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/3937e8a5a855aebc4c8e16206fd69c863f567e15&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/3937e8a5a855aebc4c8e16206fd69c863f567e15&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1048493" author="xgen-internal-githook" created="Thu, 1 Oct 2015 12:30:55 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;scotthernandez&apos;, u&apos;name&apos;: u&apos;Scott Hernandez&apos;, u&apos;email&apos;: u&apos;scotthernandez@gmail.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-18983&quot; title=&quot;Process oplog inserts, and applying, on the secondary in parallel&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-18983&quot;&gt;&lt;del&gt;SERVER-18983&lt;/del&gt;&lt;/a&gt;: enforce batch write durability in setMinvalid function&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/c4c3722e288bd13f40a5404cc20d44d077d469ca&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/c4c3722e288bd13f40a5404cc20d44d077d469ca&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1025201" author="scotthernandez" created="Wed, 9 Sep 2015 15:04:25 +0000"  >&lt;p&gt;We are moving forward with this work to improve performance during replication.&lt;/p&gt;</comment>
                            <comment id="995069" author="scotthernandez" created="Mon, 3 Aug 2015 15:12:02 +0000"  >&lt;p&gt;This work is on hold while other work is being done on inserting an array (vector) of documents. The multi-insert, at the integration+storage layer, is showing great performance improvements during testing and would remove the need for the work defined in this issue.&lt;/p&gt;</comment>
                            <comment id="953535" author="scotthernandez" created="Mon, 29 Jun 2015 19:18:54 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas%4010gen.com&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas@10gen.com&quot;&gt;bruce.lucas@10gen.com&lt;/a&gt;, I think we are fine as long as they commit in a group. The issue will be if they are committed out of order such that some are missing from the oplog, and there are entries with larger ts values than the missing entries, since that will be used as the high water mark to start applying on recovery.&lt;/p&gt;</comment>
                            <comment id="953493" author="bruce.lucas@10gen.com" created="Mon, 29 Jun 2015 18:47:08 +0000"  >&lt;p&gt;Thanks Scott. So it seems that within the current design parallelizing oplog inserts would need to be a separate step as it is now, with a separate set of worker threads. TBD how that would perform - my gut feeling is it would be ok, will give it a try. Any functional issues with that approach?&lt;/p&gt;</comment>
                            <comment id="953270" author="scotthernandez" created="Mon, 29 Jun 2015 16:04:58 +0000"  >&lt;p&gt;Bruce, we currently don&apos;t insert the oplog entries until after all database operations have completed successfully, since we use those oplog entries as markers for where to apply from on failure/recovery, if we encounter an error while applying a batch. By doing them in the worker threads I believe this will break recovery on a failed batch apply, which will result in missing/skipped oplog entries. We would need a new &quot;batchStartOptime&quot; to match the &quot;minValid&quot; (really &quot;batchEndOptime&quot;) to ensure correctly recovery. This work will be a bit more involved but follows future designs needed for writing to the oplog before/during application (&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-7200&quot; title=&quot;use oplog as op buffer on secondaries&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-7200&quot;&gt;&lt;del&gt;SERVER-7200&lt;/del&gt;&lt;/a&gt;) of those entries so the work will not be wasted, but is not minor, or without (high) risk.&lt;/p&gt;</comment>
                            <comment id="949872" author="bruce.lucas@10gen.com" created="Wed, 24 Jun 2015 21:49:30 +0000"  >&lt;p&gt;Create &lt;a href=&quot;https://jira.mongodb.org/browse/WT-1979&quot; title=&quot;Lower performance of out-of-order inserts can contribute to replication lag&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-1979&quot;&gt;&lt;del&gt;WT-1979&lt;/del&gt;&lt;/a&gt; to investigate whether the issue describe in the preceding comment can be addressed in WT.&lt;/p&gt;</comment>
                            <comment id="949757" author="bruce.lucas@10gen.com" created="Wed, 24 Jun 2015 19:58:17 +0000"  >&lt;p&gt;Identical oplog order on the primary and secondary is maintained by computing an identical RecordId from the oplog entry timestamp, rather than using the normal monotonic RecordId. When used on the primary the RecordIds are inserted nearly in monotonically increasing order, and that is a case that WT optimizes for. However if oplog entry insertion is parallelized on the secondary and each worker thread is independently inserting entries with a RecordId dictated by the oplog entry, the insertions in general will not be in order by RecordId, which is a less optimal path in WT, and this prevents parallelization of oplog inserts on the secondary from achieving performance parity with the parallel oplog inserts on the primary.&lt;/p&gt;

&lt;p&gt;To measure the impact of this, oplog entry insertion on the secondary was parallelized, and insertion using the computed RecordId was compared with insertion using the standard monotonic RecordId. (Note that using the standard monotonic RecordId on the secondary creates an invalid oplog because it is no longer in timestamp order, so this change is for performance evaluation only.) Secondary processing rate relative to primary was measured by comparing number of entries inserted into oplog on secondary with number on primary at a particular point in time during the run:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;                                               secondary/primary   relative&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;                                                                   performance&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;sequential insert, normal computed RecordId    17123525/20187676 = 0.848&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;parallel insertion, normal computed RecordId   17238547/20046073 = 0.860&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;parallel insertion, monotonic RecordId         20056367/20060385 = 0.999&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;ul&gt;
	&lt;li&gt;parallelizing oplog insertion on the secondary by itself does little if anything to achieve parity with the primary&lt;/li&gt;
	&lt;li&gt;however eliminating the out-of-order inserts into the oplog on the secondary by using a monotonic RecordId combined with parallelizing the oplog inserts achieves parity btween the primary and secondary&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;It&apos;s unclear whether this issue can be addressed in the mongod layer, or whether an improvement in WT for the out-of-order insertion case is needed.&lt;/p&gt;</comment>
                            <comment id="942150" author="schwerin" created="Tue, 16 Jun 2015 19:05:20 +0000"  >&lt;p&gt;On WiredTiger, that current implementation of the oplog is a btree, so doing the inserts out of order is OK so long as we&apos;re also prohibiting oplog reads. I&apos;m pretty sure that we do prohibit oplog reads during the oplog application process, so it should be easy to try moving the oplog writes into the threads that do the per-document update work. Should be easy to test, anyhow.&lt;/p&gt;</comment>
                            <comment id="942068" author="milkie" created="Tue, 16 Jun 2015 18:10:38 +0000"  >&lt;p&gt;We&apos;re going to experiment to see if this is an actual solution or if this is a problem that can solved in a different way.&lt;/p&gt;</comment>
                            <comment id="941614" author="milkie" created="Tue, 16 Jun 2015 12:44:00 +0000"  >&lt;p&gt;The secondary&apos;s oplog needs to match the order of the primary&apos;s oplog.  We will need help from the storage system in order to parallelize this.&lt;br/&gt;
One idea specifically for WiredTiger would be the ability to dictate the RecordId index order.&lt;br/&gt;
Also, using the bulk loader feature to insert each sorted batch of oplog entries may help.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                            <outwardlinks description="depends on">
                                        <issuelink>
            <issuekey id="231644">SERVER-20655</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="212871">WT-1979</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="228959">SERVER-20326</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is depended on by">
                                        <issuelink>
            <issuekey id="209769">SERVER-18908</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="243424">SERVER-21858</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                                        </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>17.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 16 Jun 2015 12:44:00 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        6 years, 19 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[<s><a href='https://jira.mongodb.org/browse/CS-21676'>CS-21676</a></s>, <s><a href='https://jira.mongodb.org/browse/WT-1979'>WT-1979</a></s>, <s><a href='https://jira.mongodb.org/browse/SERVER-20326'>SERVER-20326</a></s>, <s><a href='https://jira.mongodb.org/browse/SERVER-20655'>SERVER-20655</a></s>]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>false</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>mathias@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            6 years, 19 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_16465" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Linked BF Score</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>schwerin@mongodb.com</customfieldvalue>
            <customfieldvalue>backlog-server-tuning</customfieldvalue>
            <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>milkie@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>martin.bligh</customfieldvalue>
            <customfieldvalue>mathias@mongodb.com</customfieldvalue>
            <customfieldvalue>scotthernandez</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrl35b:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hreimf:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="705">QuInt A (10/12/15)</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11861" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>User Summary</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="11856"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrllbz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>