<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:29:37 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-32226] oldest_timestamp should track the last applied time, during initial sync</title>
                <link>https://jira.mongodb.org/browse/SERVER-32226</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Currently, the oldest_timestamp only tracks the stable timestamp, which tracks applied optimes as they commit. However, applied optimes only become stable timestamp candidates if the consistency mode of the node is &quot;Consistent&quot;.  During initial sync, the consistency mode of the node is not &quot;Consistent&quot;, and thus we do not add any new stable timestamp candidates while in the oplog replay phase.  This results in the oldest_timestamp not moving, which can quickly fill up cache with timestamp data during the oplog replay phase of initial sync.&lt;/p&gt;</description>
                <environment></environment>
        <key id="469944">SERVER-32226</key>
            <summary>oldest_timestamp should track the last applied time, during initial sync</summary>
                <type id="3" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14718&amp;avatarType=issuetype">Task</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="daniel.gottlieb@mongodb.com">Daniel Gottlieb</assignee>
                                    <reporter username="milkie@mongodb.com">Eric Milkie</reporter>
                        <labels>
                            <label>RF36</label>
                    </labels>
                <created>Fri, 8 Dec 2017 18:19:47 +0000</created>
                <updated>Mon, 30 Oct 2023 23:10:05 +0000</updated>
                            <resolved>Mon, 11 Dec 2017 15:26:13 +0000</resolved>
                                    <version>3.6.0</version>
                                    <fixVersion>3.6.1</fixVersion>
                    <fixVersion>3.7.1</fixVersion>
                                    <component>Replication</component>
                    <component>Storage</component>
                                        <votes>0</votes>
                                    <watches>17</watches>
                                                                                                                <comments>
                            <comment id="1748642" author="xgen-internal-githook" created="Mon, 11 Dec 2017 17:43:57 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Daniel Gottlieb&apos;, &apos;email&apos;: &apos;daniel.gottlieb@mongodb.com&apos;, &apos;username&apos;: &apos;dgottlieb&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-32226&quot; title=&quot;oldest_timestamp should track the last applied time, during initial sync&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-32226&quot;&gt;&lt;del&gt;SERVER-32226&lt;/del&gt;&lt;/a&gt;: Have the oldest timestamp track the last applied optime during initial sync.&lt;/p&gt;

&lt;p&gt;(cherry picked from commit 1a80d3e2817f758f07fa4de722ad278df4de7ac7)&lt;br/&gt;
Branch: v3.6&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/26caeb66ad2a6733313f1e9c25afb812fd4a284e&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/26caeb66ad2a6733313f1e9c25afb812fd4a284e&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1748336" author="xgen-internal-githook" created="Mon, 11 Dec 2017 14:32:50 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Daniel Gottlieb&apos;, &apos;email&apos;: &apos;daniel.gottlieb@mongodb.com&apos;, &apos;username&apos;: &apos;dgottlieb&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-32226&quot; title=&quot;oldest_timestamp should track the last applied time, during initial sync&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-32226&quot;&gt;&lt;del&gt;SERVER-32226&lt;/del&gt;&lt;/a&gt;: Have the oldest timestamp track the last applied optime during initial sync.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/1a80d3e2817f758f07fa4de722ad278df4de7ac7&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/1a80d3e2817f758f07fa4de722ad278df4de7ac7&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1747951" author="michael.cahill" created="Sun, 10 Dec 2017 22:46:54 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=daniel.gottlieb&quot; class=&quot;user-hover&quot; rel=&quot;daniel.gottlieb&quot;&gt;daniel.gottlieb&lt;/a&gt;, I think we should take your fix for this issue.&lt;/p&gt;

&lt;p&gt;In parallel, I&apos;ll work up the WiredTiger change I sketched out above and we can decide if it will be feasible.  As you noted, care will be required when the &lt;tt&gt;oldest_timestamp&lt;/tt&gt; is set for the first time, which might create more problems than it solves.&lt;/p&gt;</comment>
                            <comment id="1747882" author="milkie" created="Sun, 10 Dec 2017 14:18:38 +0000"  >&lt;p&gt;Since reads are disallowed while a node is doing an initial sync, I&apos;m not sure the &quot;readConcernMajority lag&quot; statistic makes sense.  If anything, it should read as &quot;infinity&quot; during the initial sync period.&lt;/p&gt;</comment>
                            <comment id="1747881" author="bruce.lucas@10gen.com" created="Sun, 10 Dec 2017 14:17:54 +0000"  >&lt;blockquote&gt;
&lt;p&gt;I don&apos;t have an explanation for the value of &quot;10&quot; after initial sync completes.&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;I wonder if this somehow relates to the fact that the replica set is now idle and we are writing no-ops at 10-second intervals. We could test this theory by adjusting periodicNoopIntervalSecs to see if range of timestamps pinned changes.&lt;/p&gt;</comment>
                            <comment id="1747875" author="bruce.lucas@10gen.com" created="Sun, 10 Dec 2017 13:55:39 +0000"  >&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/172911/172911_comparison.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;As expected and as you noted range of timestamps pinned is now well behaved&lt;/li&gt;
&lt;/ul&gt;


&lt;ul&gt;
	&lt;li&gt;Lookaside score is now well behaved as well, and this was one of the indicators of cache problems during initial sync. I&apos;m not sure why this is different in patch though as the amount of data in cache is about the same in both master and patch.&lt;/li&gt;
&lt;/ul&gt;


&lt;ul&gt;
	&lt;li&gt;However readConcernMajority lag (difference between readConcernMajority and applied optimes) now grows during oplog replay phase, whereas previously, aside from an initial very large spike, it was 0. We should check the ftdc from the primary and other secondary to make sure they aren&apos;t seeing the same thing: if they are that would be problematic wrt cache and would need to be fixed; if not then this is I guess in effect a reporting error for the new secondary and would be nice to fix if possible, I think.&lt;/li&gt;
&lt;/ul&gt;


&lt;ul&gt;
	&lt;li&gt;Our reported replica set lag (difference between replSetGetStats optimeDate on primary vs secondary) also grows during this phase, whereas (I believe, can check again) in 3.4 we reported decreasing lag as we applied ops and caught up. Since this provides an indicator of progress it might be nice to see if this could be fixed.&lt;/li&gt;
&lt;/ul&gt;
</comment>
                            <comment id="1747813" author="daniel.gottlieb@10gen.com" created="Sun, 10 Dec 2017 04:15:15 +0000"  >&lt;p&gt;I&apos;ve attached screenshots of FTDC data demonstrating a workload against master and a patch put together (not &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=michael.cahill&quot; class=&quot;user-hover&quot; rel=&quot;michael.cahill&quot;&gt;michael.cahill&lt;/a&gt;&apos;s idea, but simply advancing the oldest timestamp in the oplog application phase). As per the previous comments, we may not go with this patch. I just wanted to put the numbers out there to ensure there&apos;s consensus on what we&apos;re fixing.&lt;/p&gt;

&lt;p&gt;Against master we can see transactions being pinned shoot up right when documents are inserted into the oplog (the beginning of the oplog application phase). This screenshot was taken from FTDC data before the initial sync completed. My goal was to pipeline some of my work. The attached FTDC is the full timespan.&lt;br/&gt;
&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/172901/172901_PinnedMaster.png&quot; width=&quot;1000&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;Against the patch we can see the transactions being pinned stays steady while documents are inserted into the oplog. I don&apos;t have an explanation for the value of &quot;10&quot; after initial sync completes. I also don&apos;t know why I chose &quot;filtered&quot;. I believe those details are inconsequential.&lt;br/&gt;
&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/172902/172902_PinnedPatch.png&quot; width=&quot;1000&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;This wasn&apos;t a real perf test (was using non-optimized, debug builds), and in both cases the oplog application phase took rough the same time, 3m30s for master, 3m15s for the patch. Also this was an insert only workload and without constrained WT cache. &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt; I&apos;ve attached the full FTDC data from master/the patch. What we have is a pretty educated guess as to what&apos;s going on. I just want to make sure you&apos;re not looking at some additional data points in FTDC that would suggest this change of advancing the oldest timestamp would not fix the initial sync problem we&apos;re seeing.&lt;/p&gt;</comment>
                            <comment id="1747805" author="daniel.gottlieb@10gen.com" created="Sun, 10 Dec 2017 03:01:03 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=michael.cahill&quot; class=&quot;user-hover&quot; rel=&quot;michael.cahill&quot;&gt;michael.cahill&lt;/a&gt; I haven&apos;t thought of the problem that way, but you&apos;re suggestion would be much easier. One assumption we&apos;d have to vet is that the replication&apos;s initial call to set the oldest timestamp coming out of initial sync is synchronous with processing future batches. Otherwise the secondary may silently throw away history that is necessary for a consistent read of the data.&lt;/p&gt;

&lt;p&gt;On a similar note, a (future) side-effect would be for recover to a timestamp. We&apos;d have to make sure the oldest timestamp is set before oplog recovery happens on startup.&lt;/p&gt;</comment>
                            <comment id="1747804" author="michael.cahill" created="Sun, 10 Dec 2017 02:44:50 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=daniel.gottlieb&quot; class=&quot;user-hover&quot; rel=&quot;daniel.gottlieb&quot;&gt;daniel.gottlieb&lt;/a&gt;, it would be a relatively simple change to WiredTiger to have it not maintain history if no &lt;tt&gt;oldest_timestamp&lt;/tt&gt; is set, and to provide a way to clear the &lt;tt&gt;oldest_timestamp&lt;/tt&gt;.&lt;/p&gt;

&lt;p&gt;Would it be easier to say that we only set an oldest timestamp in some finite set of cases (e.g., when the repl state is PRIMARY or SECONDARY) than to explicitly track track optimes as they are applied?&lt;/p&gt;</comment>
                            <comment id="1747138" author="bruce.lucas@10gen.com" created="Fri, 8 Dec 2017 18:34:51 +0000"  >&lt;p&gt;This has been observed in some cases to cause initial sync to get permanently stuck with a full cache.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="468757">SERVER-32185</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="470028">SERVER-32236</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="1385932">SERVER-49006</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="172901" name="PinnedMaster.png" size="82611" author="daniel.gottlieb@mongodb.com" created="Sun, 10 Dec 2017 03:39:15 +0000"/>
                            <attachment id="172902" name="PinnedPatch.png" size="81969" author="daniel.gottlieb@mongodb.com" created="Sun, 10 Dec 2017 03:39:27 +0000"/>
                            <attachment id="172911" name="comparison.png" size="283423" author="bruce.lucas@mongodb.com" created="Sun, 10 Dec 2017 13:55:07 +0000"/>
                            <attachment id="172903" name="server32226.tgz" size="523482" author="daniel.gottlieb@mongodb.com" created="Sun, 10 Dec 2017 04:03:14 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>10.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="15141"><![CDATA[v3.6]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[500A000000aPFkpIAG]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 8 Dec 2017 18:34:51 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        6 years, 9 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            6 years, 9 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>daniel.gottlieb@mongodb.com</customfieldvalue>
            <customfieldvalue>milkie@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>michael.cahill@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htlvbb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|htbo1b:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="2031">Repl 2017-12-18</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htlhfb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>