<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:15:04 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-27403] Consider term and rbid when validating the proposed sync source</title>
                <link>https://jira.mongodb.org/browse/SERVER-27403</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;When the document returned by the GTE query against our sync source does not include our most recent optime (ie the term and timestamp of our most recent oplog entry), we &lt;a href=&quot;https://github.com/mongodb/mongo/blob/b07f96535de47c2da6700074e79ce0ee6eb1ae21/src/mongo/db/repl/oplog_fetcher.cpp#L153&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;currently unconditionally return OplogStartMissing&lt;/a&gt; and go into ROLLBACK.  The GTE query only includes the timestamp however, not the term, so we need to check the term and if the optime (including term) of our most recent oplog entry is higher than the optime we got back from the GTE query, we should not go into rollback but should just choose a new sync source.&lt;/p&gt;</description>
                <environment></environment>
        <key id="338476">SERVER-27403</key>
            <summary>Consider term and rbid when validating the proposed sync source</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="judah.schvimer@mongodb.com">Judah Schvimer</assignee>
                                    <reporter username="spencer@mongodb.com">Spencer Brody</reporter>
                        <labels>
                    </labels>
                <created>Tue, 13 Dec 2016 16:24:47 +0000</created>
                <updated>Wed, 29 Aug 2018 08:53:42 +0000</updated>
                            <resolved>Wed, 8 Mar 2017 22:18:15 +0000</resolved>
                                                    <fixVersion>3.2.13</fixVersion>
                    <fixVersion>3.4.4</fixVersion>
                    <fixVersion>3.5.5</fixVersion>
                                    <component>Replication</component>
                                        <votes>0</votes>
                                    <watches>16</watches>
                                                                                                                <comments>
                            <comment id="1542621" author="xgen-internal-githook" created="Thu, 6 Apr 2017 15:26:13 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;judahschvimer&apos;, u&apos;name&apos;: u&apos;Judah Schvimer&apos;, u&apos;email&apos;: u&apos;judah@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-27403&quot; title=&quot;Consider term and rbid when validating the proposed sync source&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-27403&quot;&gt;&lt;del&gt;SERVER-27403&lt;/del&gt;&lt;/a&gt; Ensure sync source is ahead and has not rolled back after first OplogFetcher batch&lt;/p&gt;

&lt;p&gt;(cherry picked from commit c05f900dd80342d0899f6461f845dc97fe942b01)&lt;br/&gt;
Branch: v3.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/6af0d685a9a5924fb982d3bfabdc5ed879f6a82c&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/6af0d685a9a5924fb982d3bfabdc5ed879f6a82c&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1532041" author="xgen-internal-githook" created="Fri, 24 Mar 2017 13:59:10 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;judahschvimer&apos;, u&apos;name&apos;: u&apos;Judah Schvimer&apos;, u&apos;email&apos;: u&apos;judah@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-27403&quot; title=&quot;Consider term and rbid when validating the proposed sync source&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-27403&quot;&gt;&lt;del&gt;SERVER-27403&lt;/del&gt;&lt;/a&gt; &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-28278&quot; title=&quot;Wait for desired sync source to replicate dummy write before running replSetSyncFrom in syncFrom shell helper&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-28278&quot;&gt;&lt;del&gt;SERVER-28278&lt;/del&gt;&lt;/a&gt; Ensure sync source is ahead and not rolled back after first fetcher batch&lt;br/&gt;
Branch: v3.2&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/45cc6d20a413d88fc49f6dac257f800fda926be6&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/45cc6d20a413d88fc49f6dac257f800fda926be6&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1519457" author="xgen-internal-githook" created="Wed, 8 Mar 2017 22:16:14 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;judahschvimer&apos;, u&apos;name&apos;: u&apos;Judah Schvimer&apos;, u&apos;email&apos;: u&apos;judah@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-27403&quot; title=&quot;Consider term and rbid when validating the proposed sync source&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-27403&quot;&gt;&lt;del&gt;SERVER-27403&lt;/del&gt;&lt;/a&gt; Ensure sync source is ahead and has not rolled back after first OplogFetcher batch&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/c05f900dd80342d0899f6461f845dc97fe942b01&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/c05f900dd80342d0899f6461f845dc97fe942b01&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1472781" author="spencer" created="Mon, 9 Jan 2017 17:32:39 +0000"  >&lt;p&gt;I think the rbid check is necessary to ensure that our sync source continues to have our minvalid after we check for it.  I do think we might be able to eliminate the proposed extra round trip in sync source resolver to get the last applied optime if we followed Siyuan&apos;s suggestion and included the lastOpApplied in the metadata and used that in the OplogFetcher::checkRemoteOplogStart() check.&lt;/p&gt;</comment>
                            <comment id="1471760" author="redbeard0531" created="Sat, 7 Jan 2017 00:52:27 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=siyuan.zhou&quot; class=&quot;user-hover&quot; rel=&quot;siyuan.zhou&quot;&gt;siyuan.zhou&lt;/a&gt; Once we establish a cursor &lt;b&gt;to the oplog&lt;/b&gt;, we can rely on all getMores on that cursor being from the same timeline because a (succesfull) rollback will truncate the oplog which kills all cursors: &lt;a href=&quot;https://github.com/mongodb/mongo/blob/ef1f1739d6cbff9fb4ddbcc77d467f183c0ab9f2/src/mongo/db/catalog/collection.cpp#L921&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/blob/ef1f1739d6cbff9fb4ddbcc77d467f183c0ab9f2/src/mongo/db/catalog/collection.cpp#L921&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;For the purposes of oplog reading we don&apos;t care if our upstream node unsuccessfully rolls back as long as it hasn&apos;t truncated the oplog.&lt;/p&gt;</comment>
                            <comment id="1471724" author="thomas.schubert" created="Fri, 6 Jan 2017 23:20:12 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=siyuan.zhou&quot; class=&quot;user-hover&quot; rel=&quot;siyuan.zhou&quot;&gt;siyuan.zhou&lt;/a&gt;, my understanding is that we close all connections &lt;b&gt;except&lt;/b&gt; those with the keepOpen property: &lt;a href=&quot;https://github.com/mongodb/mongo/blob/ba55f25/src/mongo/db/repl/replication_coordinator_impl.cpp#L2590-L2600&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/blob/ba55f25/src/mongo/db/repl/replication_coordinator_impl.cpp#L2590-L2600&lt;/a&gt;. Discussed on &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-26986&quot; title=&quot;Drop all connections when transitioning to ROLLBACK or REMOVED states&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-26986&quot;&gt;&lt;del&gt;SERVER-26986&lt;/del&gt;&lt;/a&gt;.&lt;/p&gt;</comment>
                            <comment id="1471707" author="siyuan.zhou@10gen.com" created="Fri, 6 Jan 2017 23:02:30 +0000"  >&lt;p&gt;I think we &lt;a href=&quot;https://github.com/mongodb/mongo/blob/ba55f25/src/mongo/db/repl/replication_coordinator_impl.cpp#L2590-L2600&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;close all connections on rollback&lt;/a&gt;. Closing connections (more importantly cursors) on rollback ensures that data fetched by consecutive query/getMore requests is continuous.&lt;/p&gt;

&lt;p&gt;Changing the semantic of lastVisible may be a trouble for multiversion. It&apos;s easier to add the last applied optime to metadata for rollback detection. If the new field is absent, we can fall back to the current behavior for back-compatibility in multiversion replset. Once the user finishes upgrade, the new correct logic works on all nodes.&lt;/p&gt;

&lt;p&gt;The benefit is that all rollback detection logic is at the same place.&lt;/p&gt;</comment>
                            <comment id="1467281" author="redbeard0531" created="Fri, 30 Dec 2016 23:00:04 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=siyuan.zhou&quot; class=&quot;user-hover&quot; rel=&quot;siyuan.zhou&quot;&gt;siyuan.zhou&lt;/a&gt;, we don&apos;t actually close all connections on rollback. That is why we need to use the RBID (carefully!) to ensure that rollbacks haven&apos;t happened in critical sections.&lt;/p&gt;</comment>
                            <comment id="1467277" author="siyuan.zhou@10gen.com" created="Fri, 30 Dec 2016 22:32:07 +0000"  >&lt;p&gt;As discussed with &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=judah.schvimer&quot; class=&quot;user-hover&quot; rel=&quot;judah.schvimer&quot;&gt;judah.schvimer&lt;/a&gt;, the lastVisible is not the last applied optime. It&#8217;s the max of lastOpTimeOnClient and committed snapshot optime. The discussion of lastVisible is tracked in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-27543&quot; title=&quot;Create new metadata for oplog queries&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-27543&quot;&gt;&lt;del&gt;SERVER-27543&lt;/del&gt;&lt;/a&gt;. Changing and using lastVisible needs extra work due to multiversion support.&lt;/p&gt;</comment>
                            <comment id="1467234" author="siyuan.zhou@10gen.com" created="Fri, 30 Dec 2016 21:14:30 +0000"  >&lt;p&gt;Judah&apos;s proposal sounds correct given that we close connections on rollback (so RBID serves as a version). I think there might be an easier fix though. If GTE response starts from our last fetched optime, it means all the oplog entries up to our last fetched document are the same as those on our sync source, due to Log Matching property in Raft:&lt;/p&gt;

&lt;p&gt;&lt;cite&gt;if two logs contain an entry with the same index and term, then the logs are identical in all entries up through the given index.&lt;/cite&gt;&lt;/p&gt;

&lt;p&gt;In addition, we need to compare the last applied optime on sync source and make sure it&apos;s ahead of our last fetched optime. This requirement is the same in Raft. Actually the last applied optime on sync source is ready to use in ReplSetMetadata since we &lt;a href=&quot;https://github.com/mongodb/mongo/blob/0cd2bf2/src/mongo/db/repl/oplog_fetcher.cpp#L568&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;always query it in oplog fetcher&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;For each GTE response, we make sure&lt;br/&gt;
1) (Oplog Matching) its first entry is the same as our last fetched, and &lt;br/&gt;
2) (Oplog Freshness) their last applied (ReplSetMetadata::getLastOpVisible()) is greater than our last fetched one.&lt;/p&gt;

&lt;p&gt;If Oplog Matching is false but Oplog Freshness is true, we have to roll back.&lt;br/&gt;
If Oplog Freshness is false, choose another sync source.&lt;/p&gt;

&lt;p&gt;As a result, the correctness is guaranteed by these two conditions, even if our sync source rolls back, and choosing sync source is only for performance. Maintaining RBID is not necessary in steady state replication then.&lt;/p&gt;</comment>
                            <comment id="1466824" author="judah.schvimer" created="Thu, 29 Dec 2016 22:09:33 +0000"  >&lt;p&gt;Per our discussion we will be doing the following:&lt;br/&gt;
The &lt;tt&gt;SyncSourceResolver&lt;/tt&gt; will now make sure that our sync source is safe to use by both checking the Rollback ID (RBID) of the sync source candidate and by looking at its last applied OpTime. The &lt;tt&gt;SyncSourceResolver&lt;/tt&gt; will now:&lt;br/&gt;
1. Choose a sync source as it currently does. The sync source selection algorithm will make sure that we only choose sync sources that are ahead of our last applied OpTime. &lt;br/&gt;
2. Get the sync source candidate&apos;s RBID and save it for later. Work on &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-27050&quot; title=&quot;Ensure upstream node doesn&amp;#39;t roll back after checking minvald&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-27050&quot;&gt;&lt;del&gt;SERVER-27050&lt;/del&gt;&lt;/a&gt; will do this, however it does it only if there is a minValid present, this ticket must make it an unconditional check.&lt;br/&gt;
3. Get sync source candidate&apos;s last applied OpTime and check to see it&apos;s greater than or equal to our last fetched OpTime. (This must be added by this ticket)&lt;br/&gt;
4. Get the sync source candidate&apos;s oldest oplog entry and see if we are even able to sync from this sync source.&lt;br/&gt;
5. Finally, Check if the sync source candidate contains minValid if one exists.&lt;/p&gt;

&lt;p&gt;As part of &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-27050&quot; title=&quot;Ensure upstream node doesn&amp;#39;t roll back after checking minvald&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-27050&quot;&gt;&lt;del&gt;SERVER-27050&lt;/del&gt;&lt;/a&gt;, the RBID will be checked again in BackgroundSync before the sync source is used. This check should be moved by this ticket into &lt;tt&gt;OplogFetcher::checkRemoteOplogStart()&lt;/tt&gt; so that it is performed on the first batch only and before we decide to go into Rollback.&lt;/p&gt;

&lt;p&gt;Finally, &lt;tt&gt;OplogFetcher::checkRemoteOplogStart()&lt;/tt&gt; is now mostly fine. Since our sync source cannot have rolled back since we chose it, it appears impossible for our GTE query to return empty documents. We will add an &lt;tt&gt;fassert&lt;/tt&gt; there instead of returning &lt;tt&gt;RemoteOplogStale&lt;/tt&gt;. The &lt;tt&gt;OplogStartMissing&lt;/tt&gt; cases should be fine now. If we fall off the back of the sync source&apos;s oplog between probing the sync source and getting our first batch, we will return an &lt;tt&gt;OplogStartMissing&lt;/tt&gt; error here and go into rollback, leading to an inevitable &lt;tt&gt;UnrecoverableRollbackException&lt;/tt&gt; since we will not be able to find a common point. That is a small window and users rarely fall of the back of their oplog. Further work could check if this is the case by re-querying the sync source&apos;s oldest oplog entry before going into rollback, but that is out of the scope of this ticket. &lt;/p&gt;</comment>
                            <comment id="1462439" author="spencer" created="Tue, 20 Dec 2016 19:24:48 +0000"  >&lt;p&gt;You&apos;re right, that is a problem.&lt;/p&gt;

&lt;p&gt;For reference, the original problem this ticket is trying to avoid is node A or B rolling back in the following situation&lt;/p&gt;
&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;Timestamp&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;1&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;2&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;3&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;4&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;5&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;6&lt;/th&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Node A (term)&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;3&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&amp;nbsp;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Node B (term)&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;3&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&amp;nbsp;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Node C (term)&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;2&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;2&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;2&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;2&lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;
</comment>
                            <comment id="1461546" author="siyuan.zhou@10gen.com" created="Mon, 19 Dec 2016 21:05:19 +0000"  >&lt;p&gt;That&apos;s where our implementation differs from Raft. Even if a majority of nodes know about term 2, they can still replicate logs from the old primary or nodes that synced from the old primary.&lt;/p&gt;

&lt;p&gt;Even in Raft, I still think this is possible. At the beginning, Node C steps up in term 2.&lt;/p&gt;
&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;Timestamp&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;1&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;2&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;3&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;4&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;5&lt;/th&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Node A (term)&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&amp;nbsp;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Node B (term)&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&amp;nbsp;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&amp;nbsp;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&amp;nbsp;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Node C (term)&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;2&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&amp;nbsp;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&amp;nbsp;&lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;


&lt;p&gt;Then Node A steps up again in term 3.&lt;/p&gt;
&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;Timestamp&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;1&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;2&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;3&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;4&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;5&lt;/th&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Node A (term)&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;3&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Node B (term)&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&amp;nbsp;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&amp;nbsp;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&amp;nbsp;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Node C (term)&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;2&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&amp;nbsp;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&amp;nbsp;&lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;


&lt;p&gt;Node A replicates and commits logs from old terms (term 1).&lt;/p&gt;
&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;Timestamp&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;1&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;2&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;3&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;4&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;5&lt;/th&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Node A (term)&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;3&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Node B (term)&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;3&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Node C (term)&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;2&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&amp;nbsp;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&amp;nbsp;&lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;
</comment>
                            <comment id="1461491" author="spencer" created="Mon, 19 Dec 2016 20:23:50 +0000"  >&lt;p&gt;The scenario you describe isn&apos;t possible.  For node C to have been elected in term 2, a majority of nodes must have voted for it, and thus a majority of nodes must know about term 2, and all future committed writes must happen in a term greater than or equal to 2.&lt;/p&gt;</comment>
                            <comment id="1460841" author="siyuan.zhou@10gen.com" created="Mon, 19 Dec 2016 07:33:07 +0000"  >&lt;p&gt;Should we compare our last optime against the last applied optime on our sync source rather than the first optime from the response of GLE?&lt;/p&gt;

&lt;p&gt;Imagine the following scenario, timestamps are shown as integers for simplicity. When Node C queries the oplog on Node B to find anything greater than or equal to (ts: 3), it will get (ts: 3, term 1), whose term is less than its last term in oplog  (ts: 3, term 2). But Node B is actually ahead of Node C.&lt;/p&gt;

&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;Timestamp&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;1&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;2&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;3&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;4&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;5&lt;/th&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Node A (term)&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;3&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Node B (term)&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;3&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Node C (term)&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;2&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&amp;nbsp;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&amp;nbsp;&lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;


&lt;p&gt;Having the last applied optime in GLE metadata of response and checking it against its last fetched optime is a robust way to detect rollback.&lt;/p&gt;</comment>
                            <comment id="1456871" author="spencer" created="Tue, 13 Dec 2016 21:04:41 +0000"  >&lt;p&gt;I think what we should do is break out the case where we check if the oplog entry returned by our sync source matches our most recent oplog entries into 3 cases:&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;If the sync source&apos;s entry is newer, return OplogStartMissing and go into rollback as we currently do.&lt;/li&gt;
	&lt;li&gt;If our entry is newer, we return RemoteOplogStale, and then we update bgsync to not trigger rollback on RemoteOplogStale, and to just change sync sources instead (which is basically implementing &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-26253&quot; title=&quot;Get new sync source rather than rollback on RemoteOplogStale error&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-26253&quot;&gt;&lt;del&gt;SERVER-26253&lt;/del&gt;&lt;/a&gt;).&lt;/li&gt;
	&lt;li&gt;If the two entries have the same OpTime but different hashes, fassert since that should be impossible.&lt;/li&gt;
&lt;/ol&gt;
</comment>
                            <comment id="1456748" author="milkie" created="Tue, 13 Dec 2016 19:40:04 +0000"  >&lt;p&gt;I suspect that changing the query this way may break oplogReplay, unfortunately.  Our only other option is to do a local round of filtering after we get the data back.&lt;/p&gt;</comment>
                            <comment id="1456739" author="spencer" created="Tue, 13 Dec 2016 19:33:12 +0000"  >&lt;p&gt;I believe we could do this with a change in the oplog query we do, as &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=milkie&quot; class=&quot;user-hover&quot; rel=&quot;milkie&quot;&gt;milkie&lt;/a&gt; suggests, by changing the query from&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{ts: {$gte: op.timestamp}}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;to&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{$or: [{t: {$gt: op.t}}, {$and: [{t: op.t}, {ts: {$gte: op.ts}}]}]}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;i.e. find the ops where either the term is greater than my last op, or the term is the same and the timestamp is greater.&lt;/p&gt;

&lt;p&gt;If we take this approach, then implementing &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-26253&quot; title=&quot;Get new sync source rather than rollback on RemoteOplogStale error&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-26253&quot;&gt;&lt;del&gt;SERVER-26253&lt;/del&gt;&lt;/a&gt; becomes mandatory as well or we will still trigger rollback in these situations.&lt;/p&gt;

&lt;p&gt;My bigger concern with this approach is if it will negatively impact the performance of the oplog queries.  &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=david.storch&quot; class=&quot;user-hover&quot; rel=&quot;david.storch&quot;&gt;david.storch&lt;/a&gt;, can you comment on what affect changing the query we use to fetch oplog entries for replication is likely to have?  This is the main query we use for fetching oplog entries, which runs with tailable, oplogReplay, and awaitData all set to true.&lt;/p&gt;</comment>
                            <comment id="1456410" author="milkie" created="Tue, 13 Dec 2016 16:37:36 +0000"  >&lt;p&gt;I would prefer, if possible, to construct the query such that data isn&apos;t returned if it is less than what we are looking for.  As proposed, we would do a GTE query which would leave out some, but not all, of the documents we care about not seeing, and then after getting the data we would then do another filtering pass ourselves.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                            <outwardlinks description="depends on">
                                        <issuelink>
            <issuekey id="332087">SERVER-27050</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="341625">SERVER-27543</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                                                <inwardlinks description="is duplicated by">
                                        <issuelink>
            <issuekey id="224059">SERVER-19753</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="318199">SERVER-26253</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="345299">SERVER-27672</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="353449">SERVER-27980</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="333474">SERVER-27149</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="318199">SERVER-26253</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="365731">SERVER-28377</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="372056">SERVER-28686</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="367110">SERVER-28430</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="341695">SERVER-27547</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="356068">SERVER-28068</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="363805">SERVER-28278</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="341660">SERVER-27545</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>19.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>4.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="14340"><![CDATA[v3.4]]></customfieldvalue>
    <customfieldvalue key="13440"><![CDATA[v3.2]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[500A000000VpFNOIA3, 500A000000auNPTIA2, 500A000000b7lc6IAA, 500A000000b7mGzIAI, 500A000000b8s40IAA, 500A000000b9ed9IAA, 500A000000bV9iuIAC, 500A000000bMIBdIAO, 500A000000bROp0IAG, 500A000000bST1XIAW]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 13 Dec 2016 16:37:36 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        6 years, 44 weeks, 6 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[<s><a href='https://jira.mongodb.org/browse/SERVER-27050'>SERVER-27050</a></s>, <s><a href='https://jira.mongodb.org/browse/SERVER-27543'>SERVER-27543</a></s>]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>PM-591</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>john.murphy@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            6 years, 44 weeks, 6 days ago
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_16465" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Linked BF Score</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>19.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>milkie@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>judah.schvimer@mongodb.com</customfieldvalue>
            <customfieldvalue>kelsey.schubert@mongodb.com</customfieldvalue>
            <customfieldvalue>mathias@mongodb.com</customfieldvalue>
            <customfieldvalue>siyuan.zhou@mongodb.com</customfieldvalue>
            <customfieldvalue>spencer@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrjoqf:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrb0rb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="1313">Repl 2017-01-23</customfieldvalue>
    <customfieldvalue id="1450">Repl 2017-02-13</customfieldvalue>
    <customfieldvalue id="1453">Repl 2017-03-06</customfieldvalue>
    <customfieldvalue id="1617">Repl 2017-03-27</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrifcv:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>