<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:47:14 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-37860] Only archive WT journal files after a successful MongoDB startup.</title>
                <link>https://jira.mongodb.org/browse/SERVER-37860</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Once WT starts up, it can start archiving journal files that are no longer needed. In cases where MongoDB startup fails, there may be clues from the previous run still left in the journal files that can provide some insight into how the system got into a bad state.&lt;/p&gt;</description>
                <environment></environment>
        <key id="627559">SERVER-37860</key>
            <summary>Only archive WT journal files after a successful MongoDB startup.</summary>
                <type id="2" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14711&amp;avatarType=issuetype">New Feature</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="gregory.noma@mongodb.com">Gregory Noma</assignee>
                                    <reporter username="daniel.gottlieb@mongodb.com">Daniel Gottlieb</reporter>
                        <labels>
                    </labels>
                <created>Thu, 1 Nov 2018 13:14:36 +0000</created>
                <updated>Sun, 29 Oct 2023 22:26:59 +0000</updated>
                            <resolved>Wed, 24 Jun 2020 15:07:34 +0000</resolved>
                                                    <fixVersion>4.7.0</fixVersion>
                                    <component>Storage</component>
                                        <votes>1</votes>
                                    <watches>8</watches>
                                                                                                                <comments>
                            <comment id="3222025" author="xgen-internal-githook" created="Wed, 24 Jun 2020 15:06:50 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Gregory Noma&apos;, &apos;email&apos;: &apos;gregory.noma@gmail.com&apos;, &apos;username&apos;: &apos;gregorynoma&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-37860&quot; title=&quot;Only archive WT journal files after a successful MongoDB startup.&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-37860&quot;&gt;&lt;del&gt;SERVER-37860&lt;/del&gt;&lt;/a&gt; Retain some WT log files on startup from previous runs when testing is enabled&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/4b7e5ea24f29e13d13a17c9fa2ba88b542045ed6&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/4b7e5ea24f29e13d13a17c9fa2ba88b542045ed6&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3153157" author="sue.loverso" created="Mon, 1 Jun 2020 19:05:15 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=daniel.gottlieb&quot; class=&quot;user-hover&quot; rel=&quot;daniel.gottlieb&quot;&gt;daniel.gottlieb&lt;/a&gt; this is a fortuitous time to resuscitate this discussion. &lt;/p&gt;

&lt;p&gt;There are two &lt;tt&gt;debug_mode&lt;/tt&gt; settings that might be useful to you. One was only added last week. In &lt;a href=&quot;https://jira.mongodb.org/browse/WT-6302&quot; title=&quot;Move format operations tracing into WiredTiger log files&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-6302&quot;&gt;&lt;del&gt;WT-6302&lt;/del&gt;&lt;/a&gt;, we added a &lt;tt&gt;debug_mode=(log_retention=#)&lt;/tt&gt; API for debugging. Here&apos;s the description from &lt;tt&gt;dist/api_data.py&lt;/tt&gt;:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;            adjust log archiving to retain at least this number of log files, ignored if set to 0.&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;            (Warning: this option can remove log files required for recovery if no checkpoints&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;            have yet been done and the number of log files exceeds the configured value. As&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;            WiredTiger cannot detect the difference between a system that has not yet checkpointed&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;            and one that will never checkpoint, it might discard log files before any checkpoint is&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;            done.&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;So if you can know that &quot;recent&quot; logs would be helpful, particularly for testing/debugging, you can set this as an archiving lag.&lt;/p&gt;

&lt;p&gt;Similarly, you can not worry about how many log files are needed and just keep the logs for the last N checkpoints with &lt;tt&gt;debug_mode=(checkpoint_retention=#&lt;/tt&gt;}.&lt;/p&gt;</comment>
                            <comment id="3150804" author="daniel.gottlieb@10gen.com" created="Mon, 1 Jun 2020 18:09:18 +0000"  >&lt;p&gt;We now have BFs where this would help. Particularly now that we capture operations as no-ops in the WT journal, the journal has regained its utility and also because out of order timestamps (ghost timestamps) are no longer supported by the implementation.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sue.loverso&quot; class=&quot;user-hover&quot; rel=&quot;sue.loverso&quot;&gt;sue.loverso&lt;/a&gt; mentioned a path to making this happen:&lt;/p&gt;
&lt;blockquote&gt;
&lt;p&gt;If MongoDB rapidly checkpoints during its startup then yes, archiving will happen but MongoDB is in control of when archiving will happen based on when checkpoint is called the first time.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;I&apos;m certain that&apos;s not part of the public contract WT provides, but if it&apos;s still true, that should satisfy the BF problem. Ideally we&apos;d prevent checkpointing/truncation until the server is listening on a port. I&apos;m curious if that would cause problems. Replication recovery does do a bunch of writes, but the oldest/stable timestamp is not advanced at startup. I know that for WT, the process of doing a checkpoint is &quot;good hygiene&quot;, but I don&apos;t know if that also requires oldest/stable moving.&lt;/p&gt;</comment>
                            <comment id="2048942" author="daniel.gottlieb@10gen.com" created="Thu, 1 Nov 2018 15:01:56 +0000"  >&lt;blockquote&gt;
&lt;p&gt;We could entertain some kind of configuration settings where the slot buffers don&apos;t use the file size in the calculation and force the default size or force reconfiguration to change the file size but not the buffers. &lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;Our use case would be for when the application is relatively idle (i.e: before users can connect, but after some internal background threads get kicked off). Additionally, I don&apos;t think it&apos;d be a problem to incur a reallocation cost at this stage in the process lifetime.&lt;/p&gt;

&lt;blockquote&gt;
&lt;p&gt;If MongoDB rapidly checkpoints during its startup then yes, archiving will happen but MongoDB is in control of when archiving will happen based on when checkpoint is called the first time.&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;MongoDB does typically checkpoint right at startup. That&apos;s a clever way to avoid the need to reconfigure the archiving settings! We&apos;d still have to be careful about accumulating to many large, empty WT journal files.&lt;/p&gt;</comment>
                            <comment id="2048866" author="sue.loverso" created="Thu, 1 Nov 2018 14:23:53 +0000"  >&lt;p&gt;Based on comments in the code (thank you 3-years-ago me), the reason &lt;tt&gt;file_max&lt;/tt&gt; is not reconfigurable is because the memory in the log slot buffers may be based on the log file size and we don&apos;t want to reallocate that memory on a running system. Reallocating that memory would be a pretty large undertaking in that the logging subsystem can be in use during a &lt;tt&gt;reconfig&lt;/tt&gt; so reallocating them would be disruptive. Specifically, if a small log file size is given, then the buffers are smaller (whereas they&apos;re a constant/default with the 100Mb size).&lt;/p&gt;

&lt;p&gt;That is the current state of the code. We could entertain some kind of configuration settings where the slot buffers don&apos;t use the file size in the calculation and force the default size or force reconfiguration to change the file size but not the buffers. We could consider a force option that does force reallocating the slot buffers and blocking out the logging subsystem during that time.&lt;/p&gt;

&lt;p&gt;Also, based on your ticket description, WT already does what you describe. It does not archive after the checkpoint that is part of recovery. It starts archiving after the first post-wiredtiger_open checkpoint completes. If MongoDB rapidly checkpoints during its startup then yes, archiving will happen but MongoDB is in control of when archiving will happen based on when checkpoint is called the first time.&lt;/p&gt;</comment>
                            <comment id="2048789" author="daniel.gottlieb@10gen.com" created="Thu, 1 Nov 2018 13:37:25 +0000"  >&lt;p&gt;I expect this ticket has a WT component that may be a bit challenging. I believe WT has to use a new journal file for every restart (and each file is currently always 100MB for MongoDB). When automation keeps restarting a node that&apos;s failing at startup, that would add up quickly for files that are mostly zeroes. WT does allow reconfiguring the connection to turn archiving and preallocating journal files on/off, but the &lt;tt&gt;file_max&lt;/tt&gt; can only be passed in on &lt;tt&gt;wiredtiger_open&lt;/tt&gt;. The path I see to implementing this is if the file size could be dynamically adjusted.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sue.loverso&quot; class=&quot;user-hover&quot; rel=&quot;sue.loverso&quot;&gt;sue.loverso&lt;/a&gt; would that be a feasible change? Alternatively, do you have any other ideas to help MongoDB control how the WT log files grow/get archived?&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                            <outwardlinks description="depends on">
                                        <issuelink>
            <issuekey id="1381857">WT-6428</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                                        </outwardlinks>
                                                                <inwardlinks description="is related to">
                                                        </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>6.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[500A000000cDKmEIAW]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Thu, 1 Nov 2018 14:23:53 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        3 years, 33 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[<s><a href='https://jira.mongodb.org/browse/WT-6428'>WT-6428</a></s>]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            3 years, 33 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_16465" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Linked BF Score</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>15.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>daniel.gottlieb@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>gregory.noma@mongodb.com</customfieldvalue>
            <customfieldvalue>sue.loverso@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hubt8v:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr548v:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="3937">Execution Team 2020-06-15</customfieldvalue>
    <customfieldvalue id="3938">Execution Team 2020-06-29</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hubfi7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>