<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:44:36 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-36970] Mongodb failed to start after a disk 100% full problem</title>
                <link>https://jira.mongodb.org/browse/SERVER-36970</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;I encountered a similar problem as this post:&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-26237&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;https://jira.mongodb.org/browse/SERVER-26237&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;and want to know if anybody can help on that.&#160;&lt;/p&gt;

&lt;p&gt;The mongodb starting log is listed:&lt;/p&gt;

&lt;p&gt;2018-08-31T11:32:42.803+0800 I CONTROL &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; options: { config: &quot;/etc/mongod.conf&quot;, net: &lt;/p&gt;
{ port: 27017 }
&lt;p&gt;, replication: { oplogSizeMB: 1024, replSetName: &quot;workaioplog&quot; }, security: { authorization: &quot;enabled&quot; }, storage: { dbPath: &quot;/var/lib/mongodb&quot;, journal: &lt;/p&gt;
{ enabled: false }
&lt;p&gt; }, systemLog: { destination: &quot;file&quot;, logAppend: true, path: &quot;/var/log/mongodb/mongod.log&quot; } }&lt;br/&gt;
2018-08-31T11:32:42.835+0800 I - &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Detected data files in /var/lib/mongodb created by the &apos;wiredTiger&apos; storage engine, so setting the active storage engine to &apos;wiredTiger&apos;.&lt;br/&gt;
2018-08-31T11:32:42.848+0800 I STORAGE &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Detected WT journal files. Running recovery from last checkpoint.&lt;br/&gt;
2018-08-31T11:32:42.848+0800 I STORAGE &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; journal to nojournal transition config: create,cache_size=1G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),&lt;br/&gt;
2018-08-31T11:32:42.885+0800 E STORAGE &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; WiredTiger (0) &lt;span class=&quot;error&quot;&gt;&amp;#91;1535686362:885029&amp;#93;&lt;/span&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;899:0x7f49f1f32cc0&amp;#93;&lt;/span&gt;, &lt;a href=&quot;file:WiredTiger.wt&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:WiredTiger.wt&lt;/a&gt;, connection: read checksum error for 4096B block at offset 61440: block header checksum of 3027734606 doesn&apos;t match expected checksum of 2157011047&lt;br/&gt;
2018-08-31T11:32:42.885+0800 E STORAGE &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; WiredTiger (0) &lt;span class=&quot;error&quot;&gt;&amp;#91;1535686362:885098&amp;#93;&lt;/span&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;899:0x7f49f1f32cc0&amp;#93;&lt;/span&gt;, &lt;a href=&quot;file:WiredTiger.wt&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:WiredTiger.wt&lt;/a&gt;, connection: WiredTiger.wt: encountered an illegal file format or internal value&lt;br/&gt;
2018-08-31T11:32:42.885+0800 E STORAGE &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; WiredTiger (-31804) &lt;span class=&quot;error&quot;&gt;&amp;#91;1535686362:885129&amp;#93;&lt;/span&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;899:0x7f49f1f32cc0&amp;#93;&lt;/span&gt;, &lt;a href=&quot;file:WiredTiger.wt&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:WiredTiger.wt&lt;/a&gt;, connection: the process must exit and restart: WT_PANIC: WiredTiger library panic&lt;br/&gt;
2018-08-31T11:32:42.885+0800 I - &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Fatal Assertion 28558&lt;br/&gt;
2018-08-31T11:32:42.885+0800 I - &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;***aborting after fassert() failure&lt;/p&gt;

&lt;p&gt;Can anyone help to generate the repair file for me? I attached the files probably you need in the following:&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;</description>
                <environment></environment>
        <key id="598249">SERVER-36970</key>
            <summary>Mongodb failed to start after a disk 100% full problem</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="nick.brewer">Nick Brewer</assignee>
                                    <reporter username="lorrie">Xing Fang</reporter>
                        <labels>
                            <label>envm</label>
                            <label>rfi</label>
                            <label>rps</label>
                            <label>trcf</label>
                            <label>wtc</label>
                    </labels>
                <created>Fri, 31 Aug 2018 21:08:51 +0000</created>
                <updated>Tue, 4 Sep 2018 20:00:49 +0000</updated>
                            <resolved>Tue, 4 Sep 2018 20:00:49 +0000</resolved>
                                    <version>3.2.6</version>
                                                                        <votes>0</votes>
                                    <watches>6</watches>
                                                                                                                <comments>
                            <comment id="1992964" author="nick.brewer" created="Tue, 4 Sep 2018 20:00:36 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=lorrie&quot; class=&quot;user-hover&quot; rel=&quot;lorrie&quot;&gt;lorrie&lt;/a&gt; Glad to hear it worked! Due to the risk inherent in manipulating WiredTiger files, we don&apos;t publicly offer the tool that is used to perform this particular repair.&lt;/p&gt;

&lt;p&gt;Some considerations to help mitigate issues caused by unreliable storage layers or server failures:&#160;&lt;/p&gt;
&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;Make sure your &lt;a href=&quot;https://docs.mongodb.com/manual/administration/production-notes/#disk-and-storage-systems&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;underlying storage is configured in an optimal way&lt;/a&gt;.&lt;/li&gt;
	&lt;li&gt;Schedule and perform regular checks of the integrity of your filesystems and disks.&lt;/li&gt;
	&lt;li&gt;Make sure to update MongoDB to the &lt;a href=&quot;https://docs.mongodb.com/manual/release-notes/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;most recent version&lt;/a&gt;.&lt;/li&gt;
	&lt;li&gt;Never manipulate the underlying database files in any way while &lt;tt&gt;mongod&lt;/tt&gt; is running.&lt;/li&gt;
	&lt;li&gt;Always keep up to date &lt;a href=&quot;https://docs.mongodb.com/manual/core/backups/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;backups of your databases&lt;/a&gt; and verify that you have a process in place to restore them.&lt;/li&gt;
	&lt;li&gt;Use a &lt;a href=&quot;https://docs.mongodb.com/manual/replication/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;replica set&lt;/a&gt; for improved reliability.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;-Nick&lt;/p&gt;</comment>
                            <comment id="1992950" author="lorrie" created="Tue, 4 Sep 2018 19:49:43 +0000"  >&lt;p&gt;Nick,&#160; thank you very much! It works! This looks like rocket science! You are a magician!&lt;/p&gt;

&lt;p&gt;BTW, can you share more information for how to repair it?&lt;/p&gt;</comment>
                            <comment id="1992365" author="nick.brewer" created="Tue, 4 Sep 2018 13:58:40 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=lorrie&quot; class=&quot;user-hover&quot; rel=&quot;lorrie&quot;&gt;lorrie&lt;/a&gt; I&apos;ve uploaded the files after a repair attempt - could you substitute them for the files currently in your &lt;tt&gt;dbpath&lt;/tt&gt;, and let us know if this resolves the issue?&lt;/p&gt;

&lt;p&gt;I should note that we do not recommend manipulating &lt;tt&gt;.wt&lt;/tt&gt; files with the &lt;tt&gt;wt&lt;/tt&gt; binary directly - if you have a backup of the &lt;tt&gt;dbpath&lt;/tt&gt; before you began making changes to it in this way, I would recommend using that backup when you attempt to start the mongod with the files I&apos;ve provided. &lt;/p&gt;

&lt;p&gt;-Nick&lt;/p&gt;

&lt;p&gt; &lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/attachment/195282/195282_repair-attempt-36970.tar.gz&quot; title=&quot;repair-attempt-36970.tar.gz attached to SERVER-36970&quot;&gt;repair-attempt-36970.tar.gz&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.mongodb.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt; &lt;/p&gt;</comment>
                            <comment id="1991304" author="lorrie" created="Fri, 31 Aug 2018 22:14:19 +0000"  >&lt;p&gt;Updates:&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;I installed the wt tool following this link : &quot;http://www.alexbevi.com/blog/2016/02/10/recovering-a-wiredtiger-collection-from-a-corrupt-mongodb-installation/?spm=a2c4e.11153940.blogcont73203.10.6ee15112TPYryA&quot;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;and run cmd:&lt;/p&gt;

&lt;p&gt;./wt -v -h ../mongo-bak -C &quot;extensions=&lt;span class=&quot;error&quot;&gt;&amp;#91;./ext/compressors/snappy/.libs/libwiredtiger_snappy.so&amp;#93;&lt;/span&gt;&quot; -R salvage WiredTiger.wt&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;but got err:&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;1535753498:720382]&lt;span class=&quot;error&quot;&gt;&amp;#91;28452:0x7f4f9ada3740&amp;#93;&lt;/span&gt;, &lt;a href=&quot;file:WiredTiger.wt&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:WiredTiger.wt&lt;/a&gt;, connection: read checksum error for 4096B block at offset 61440: block header checksum of 3027734606 doesn&apos;t match expected checksum of 2157011047&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;1535753498:720437&amp;#93;&lt;/span&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;28452:0x7f4f9ada3740&amp;#93;&lt;/span&gt;, &lt;a href=&quot;file:WiredTiger.wt&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:WiredTiger.wt&lt;/a&gt;, connection: WiredTiger.wt: encountered an illegal file format or internal value&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;1535753498:720468&amp;#93;&lt;/span&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;28452:0x7f4f9ada3740&amp;#93;&lt;/span&gt;, &lt;a href=&quot;file:WiredTiger.wt&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:WiredTiger.wt&lt;/a&gt;, connection: the process must exit and restart: WT_PANIC: WiredTiger library panic&lt;/p&gt;

&lt;p&gt;lt-wt: WT_PANIC: WiredTiger library panic&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Looks like it cannot repair this damaged&#160;WiredTiger.wt file.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;</comment>
                            <comment id="1991292" author="lorrie" created="Fri, 31 Aug 2018 21:46:25 +0000"  >&lt;p&gt;BTW, actually I m not sure if WireTiger.wt is the damaged wt file for repairing.&lt;/p&gt;

&lt;p&gt;From my starting log it shows &quot;&lt;b&gt;WiredTiger.wt&lt;/b&gt;&quot;, however, when I googled the similar issue, the log always provide a wt file which format is like &quot;collection-xxx&#8211;xxxx.wt&quot;, which I have a bunch under my mongodb folder.&lt;/p&gt;

&lt;p&gt;Do you have any clues?&lt;/p&gt;

&lt;p&gt;Thanks!&lt;/p&gt;</comment>
                            <comment id="1991285" author="lorrie" created="Fri, 31 Aug 2018 21:35:38 +0000"  >&lt;p&gt;Nick:&lt;/p&gt;

&lt;p&gt;Thank you very much!&lt;/p&gt;
&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;What operating system is MongoDB running on?&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Ans: it is an&#160;Ubuntu 14.04.4 x86_64 LTS&lt;/p&gt;
&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;What is the environment (virtual machine, container, etc)?&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Ans: It is a Aliyun cloud ECS server (like an AWS EC2)&lt;/p&gt;</comment>
                            <comment id="1991281" author="nick.brewer" created="Fri, 31 Aug 2018 21:26:40 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=lorrie&quot; class=&quot;user-hover&quot; rel=&quot;lorrie&quot;&gt;lorrie&lt;/a&gt; I can attempt to repair these files - however beforehand I&apos;d need to confirm:&lt;/p&gt;

&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;What operating system is MongoDB running on?&lt;/li&gt;
	&lt;li&gt;What is the environment (virtual machine, container, etc)?&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Thanks, &lt;br/&gt;
Nick&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="195191" name="logfile.tar.gz" size="92409" author="lorrie" created="Fri, 31 Aug 2018 21:04:00 +0000"/>
                            <attachment id="195282" name="repair-attempt-36970.tar.gz" size="92383" author="nick.brewer" created="Tue, 4 Sep 2018 13:58:35 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>7.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 31 Aug 2018 21:26:40 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        5 years, 23 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>nick.brewer</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            5 years, 23 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10020"><![CDATA[Linux]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>nick.brewer</customfieldvalue>
            <customfieldvalue>lorrie</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hu6x53:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|htxfef:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hu6jef:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>