<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:03:06 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-23346] WiredTiger.wt File corrupted (yet another one)</title>
                <link>https://jira.mongodb.org/browse/SERVER-23346</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;I am running mongodb 3.2.4 in a Docker container hosted in Kubernetes cluster.&lt;/p&gt;

&lt;p&gt;After restarting a kubernetes node restart, mongodb failed to start.&lt;br/&gt;
I have tried the repair command&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongod --port 27000 --dbpath /data/dbfix --repair&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;But I am still getting the error below.&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2016-03-25T07:59:26.865+0000 E STORAGE  [initandlisten] WiredTiger (-31802) [1458892766:865859][78:0x7f191a3afc80], file:WiredTiger.wt, connection: unable to read root page from file:WiredTiger.wt: WT_ERROR: non-specific WiredTiger error&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2016-03-25T07:59:26.866+0000 E STORAGE  [initandlisten] WiredTiger (0) [1458892766:866236][78:0x7f191a3afc80], file:WiredTiger.wt, connection: WiredTiger has failed to open its metadata&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2016-03-25T07:59:26.866+0000 E STORAGE  [initandlisten] WiredTiger (0) [1458892766:866365][78:0x7f191a3afc80], file:WiredTiger.wt, connection: This may be due to the database files being encrypted, being from an older version or due to corruption on disk&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2016-03-25T07:59:26.866+0000 E STORAGE  [initandlisten] WiredTiger (0) [1458892766:866607][78:0x7f191a3afc80], file:WiredTiger.wt, connection: You should confirm that you have opened the database with the correct options including all encryption and compression options&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</description>
                <environment></environment>
        <key id="275035">SERVER-23346</key>
            <summary>WiredTiger.wt File corrupted (yet another one)</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="-1">Unassigned</assignee>
                                    <reporter username="ck-lee">CK Lee</reporter>
                        <labels>
                            <label>docker</label>
                            <label>envc</label>
                            <label>rge</label>
                            <label>rpu</label>
                            <label>trcf</label>
                            <label>wtc</label>
                    </labels>
                <created>Fri, 25 Mar 2016 08:02:30 +0000</created>
                <updated>Mon, 13 Aug 2018 20:57:23 +0000</updated>
                            <resolved>Wed, 13 Apr 2016 16:55:42 +0000</resolved>
                                    <version>3.2.4</version>
                                                    <component>WiredTiger</component>
                                        <votes>0</votes>
                                    <watches>9</watches>
                                                                                                                <comments>
                            <comment id="1525525" author="bash" created="Thu, 16 Mar 2017 07:38:36 +0000"  >&lt;p&gt;Hi David,&lt;br/&gt;
thanks,&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;You need the entire dbpath as it was when you took the backup. MongoDB with WiredTiger performs consistency checks on boot by attempting to open each collection within the system. The list of collections is stored in both the WiredTiger.wt and _mdb_catalog.wt files, which also have integrity checking. If any one of the collections in the system at time of backup is not present or fails initial checksum validation then your instance will abort during boot.&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;this is verified, I tried copying files related to a particular  collection ( &lt;tt&gt;collection-&lt;b&gt;.wt&lt;/tt&gt; ,&lt;tt&gt;index-&lt;/b&gt;.wt&lt;/tt&gt; file and &lt;tt&gt;_mdb_catalog.wt&lt;/tt&gt; file ) but it fails.&lt;/p&gt;

&lt;p&gt;But now I take backup of full dbpath and if i restore it back to the same database even after few changes in the database like dropping a collection, everything works fine all processes up and running but the moment I try any partial restore(I mostly try collection restore) by copying few files from backup two things happen&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;processes won&apos;t come up due to error(Now i know the error is due to consistency checks at process boot ,thanks to David.)&lt;/li&gt;
	&lt;li&gt;(this happens when I copy &lt;tt&gt;collection-&lt;b&gt;.wt&lt;/tt&gt; ,&lt;tt&gt;index-&lt;/b&gt;.wt&lt;/tt&gt; file related to that collection and                  {{ _mdb_catalog.wt}} file along with &lt;tt&gt;Wiredtiger* .wt&lt;/tt&gt; files to dppath as restore) processes come up real fine but my database won&apos;t show up and I can see all information about that database in config database. but not in everywhere&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;&lt;em&gt;Note: I am using sharded cluster but I tried for  a stand alone mongod process too same things happen.&lt;/em&gt;&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;If you wish to restore a single collection, then you would need to first restore the whole dbpath into a stand-alone instance, then use database commands to drop the unwanted collections&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt; I want to restore it back to the same database also logically  dropping all unwanted collections would be real pain and not a flexible solution don&apos;t you think? &lt;img class=&quot;emoticon&quot; src=&quot;https://jira.mongodb.org/images/icons/emoticons/smile.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/p&gt;</comment>
                            <comment id="1525365" author="david.hows" created="Thu, 16 Mar 2017 02:22:19 +0000"  >&lt;p&gt;Hi Basharat,&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;I was trying to do restore of a specific collection with the use of its data file I wanted to know which files are related to a particular collection which needs to backup in order to get that collection back(restored).&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;You need the entire dbpath as it was when you took the backup. MongoDB with WiredTiger performs consistency checks on boot by attempting to open each collection within the system. The list of collections is stored in both the &lt;tt&gt;WiredTiger.wt&lt;/tt&gt; and &lt;tt&gt;_mdb_catalog.wt&lt;/tt&gt; files, which also have integrity checking. If any one of the collections in the system at time of backup is not present or fails initial checksum validation then your instance will abort during boot.&lt;/p&gt;

&lt;p&gt;If you wish to restore a single collection, then you would need to first restore the whole dbpath into a stand-alone instance, then use database commands to drop the unwanted collections.&lt;/p&gt;</comment>
                            <comment id="1524331" author="bash" created="Wed, 15 Mar 2017 06:56:03 +0000"  >&lt;p&gt;First of all thank  you very much for this information, very less information is available about this.( I am looking forward to more of it if you could point me to any documents related to this, would be really nice.)&lt;/p&gt;

&lt;p&gt;I was trying to do  restore of a specific collection with the use of its data file I wanted to know which files are related to a particular collection which needs to backup in order to get that collection back(restored).&lt;/p&gt;

&lt;p&gt;i tried copying &lt;tt&gt;collection-*.wt&lt;/tt&gt;, its related index file along with {{ _mdb_catalog.wt}} file . but it doesn&apos;t work.&lt;/p&gt;</comment>
                            <comment id="1524281" author="alexander.gorrod" created="Wed, 15 Mar 2017 04:20:19 +0000"  >&lt;p&gt;&lt;tt&gt;WiredTiger.wt&lt;/tt&gt; Contains information that tracks the state of the different tables in a WiredTiger database, including information about the most recent stable written data. WiredTiger flushes that file to disk regularly because it is necessary to be certain about it&apos;s contents in order to re-open the database safely. To translate that into MongoDB - each MongoDB instance has a single WiredTiger database behind it, and at the moment each collection and index resides in a different WiredTiger table.&lt;/p&gt;

&lt;p&gt;&lt;tt&gt;WiredTiger.turtle&lt;/tt&gt; Contains the metadata for the &lt;tt&gt;WiredTiger.wt&lt;/tt&gt; metadata file, i.e: Where &lt;tt&gt;WiredTiger.wt&lt;/tt&gt; contains information about the content of each table in a database, &lt;tt&gt;WiredTiger.turtle&lt;/tt&gt; contains information about the content of &lt;tt&gt;WiredTiger.wt&lt;/tt&gt;&lt;/p&gt;

&lt;p&gt;&lt;tt&gt;sizeStorer.wt&lt;/tt&gt; is a WiredTiger table populated by MongoDB that contains information about the size of collections and indexes. This is done as an optimization - because it is expensive to retrieve accurate document counts and size from WiredTiger (or most storage engines).&lt;/p&gt;
</comment>
                            <comment id="1524272" author="bash" created="Wed, 15 Mar 2017 04:00:43 +0000"  >&lt;p&gt;more information on wiredtiger.wt, wiredtiger.turtle,sizestorer.wt would be a big help.&lt;/p&gt;</comment>
                            <comment id="1238286" author="alexander.gorrod" created="Mon, 18 Apr 2016 06:27:52 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=chaokoon%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;chaokoon@gmail.com&quot;&gt;chaokoon@gmail.com&lt;/a&gt;&lt;/p&gt;

&lt;blockquote&gt;&lt;p&gt;My observation is that WiredTiger.wt (metadata) corrupts a lot more often than the collection.wt. But I&apos;m not sure whether that is worth looking into or it is just unique to my case?&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;I believe I can explain why you see corruption in the WiredTiger metadata more often than in other collections. The WiredTiger metadata needs to be flushed to stable storage each time a change is made to add or remove a collection/index as well as periodically for other reasons, wheras other collections only have their data flushed periodically. So the metadata is likely to be receiving a lot of flush operations. If the underlying disk storage subsystem does not provide reliable flush operations - corruption is more likely to occur in the metadata file than in the collection and index files.&lt;/p&gt;</comment>
                            <comment id="1237820" author="ck-lee" created="Sat, 16 Apr 2016 03:22:28 +0000"  >&lt;p&gt;For manual recovery of WiredTiger.wt, these are my steps to recover. For reference to other users who faced this similar issue and have no better ways to recover the database.&lt;br/&gt;
I manage to recover my database this way without losing any data.&lt;/p&gt;

&lt;p&gt;1) Make a backup of your corrupted database first if you have not already done so. &lt;b&gt;This recovery is very risky as it may wipe out your entire database.&lt;/b&gt;&lt;br/&gt;
2) Obtain the latest working copy of WiredTiger.wt and WiredTiger.turtle from your backup. (Note down the updated time for the files)&lt;br/&gt;
3) Install wiredtiger-2.7.0 with snappy plugin see &lt;a href=&quot;http://www.alexbevi.com/blog/2016/02/10/recovering-a-wiredtiger-collection-from-a-corrupt-mongodb-installation/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://www.alexbevi.com/blog/2016/02/10/recovering-a-wiredtiger-collection-from-a-corrupt-mongodb-installation/&lt;/a&gt;&lt;br/&gt;
4) Override the working copy of WiredTiger.wt and WiredTiger.turtle to your dbpath&lt;br/&gt;
5) List the collection .wt files that have been modified since the last updated date of WiredTiger.wt backup file. &lt;br/&gt;
5) Run wt salvage only for the collection .wt files that have been modified since the last updated date of the backup file. &lt;br/&gt;
     eg: &lt;tt&gt;./wt -v -h ../data -C &quot;extensions=&lt;span class=&quot;error&quot;&gt;&amp;#91;./ext/compressors/snappy/.libs/libwiredtiger_snappy.so&amp;#93;&lt;/span&gt;&quot; -R load -f ../collection.dump -r collection-2-880383588247732034&lt;/tt&gt;&lt;br/&gt;
6) Run &lt;tt&gt;mongod --repair&lt;/tt&gt; to fix checksum error of sizeStorer.wt and rebuild the indices.&lt;/p&gt;


</comment>
                            <comment id="1237818" author="ck-lee" created="Sat, 16 Apr 2016 03:00:57 +0000"  >&lt;p&gt;Thanks &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=anonymous.user&quot; class=&quot;user-hover&quot; rel=&quot;anonymous.user&quot;&gt;anonymous.user&lt;/a&gt;. I have restore from my backup and migrated the database from Kubernetes cluster to run on a docker with the volume mounted locally based on your recommendation. I also downgraded to 3.2.3.&lt;/p&gt;

&lt;p&gt;I have tried to reproduce the scenario by terminating the pod unexpectedly, but it wasn&apos;t very consistent.. My observation is that WiredTiger.wt (metadata) corrupts a lot more often than the collection.wt. But I&apos;m not sure whether that is worth looking into or it is just unique to my case?&lt;/p&gt;</comment>
                            <comment id="1234816" author="thomas.schubert" created="Wed, 13 Apr 2016 16:33:57 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=ck-lee&quot; class=&quot;user-hover&quot; rel=&quot;ck-lee&quot;&gt;ck-lee&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Unfortunately, this behavior indicates that the corruption was not limited to the WiredTiger.wt file. In this circumstance, my best recommendation would be to restore using your last good back up.&lt;/p&gt;

&lt;p&gt;There is very little any database can do about disk-level corruption, other than detect there is a problem. In particular, this issue appears to be the result of using NFS improperly. I would strongly recommend considering a different setup with local storage.&lt;/p&gt;

&lt;p&gt;If you do choose to continue to run on this environment, I would recommend at least setting up a replica set, which would provide a level of fault tolerance against the loss of a single database server.&lt;/p&gt;

&lt;p&gt;For MongoDB-related support discussion please post on the &lt;a href=&quot;http://groups.google.com/group/mongodb-user&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;mongodb-users group&lt;/a&gt; or &lt;a href=&quot;http://stackoverflow.com/questions/tagged/mongodb&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;Stack Overflow with the &lt;tt&gt;mongodb&lt;/tt&gt; tag&lt;/a&gt;. Questions regarding MongoDB setup would be best posted on the mongodb-users group.&lt;/p&gt;

&lt;p&gt;Kind regards,&lt;br/&gt;
Thomas&lt;/p&gt;</comment>
                            <comment id="1217687" author="ck-lee" created="Tue, 29 Mar 2016 08:17:13 +0000"  >&lt;p&gt;Thanks Ramon. I am still having issues when I try to start or repair the database. I will attach the screen dump. When I did the --repair, it started to drop all my collections.&lt;/p&gt;</comment>
                            <comment id="1217028" author="ramon.fernandez" created="Mon, 28 Mar 2016 18:30:40 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=ck-lee&quot; class=&quot;user-hover&quot; rel=&quot;ck-lee&quot;&gt;ck-lee&lt;/a&gt;, I&apos;ve uploaded the &lt;tt&gt;repair_attempt.tgz&lt;/tt&gt; file with the result of a repair attempt. Please extract it on your &lt;tt&gt;dbpath&lt;/tt&gt; and try again.&lt;/p&gt;

&lt;p&gt;Please note the &lt;a href=&quot;https://docs.mongodb.org/manual/administration/production-notes/#remote-filesystems&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;limitations for running on NFS in our documentation&lt;/a&gt;, and make sure your provider follows the necessary requirements.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-19815&quot; title=&quot;Improved mongod --repair option for WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-19815&quot;&gt;&lt;del&gt;SERVER-19815&lt;/del&gt;&lt;/a&gt; is open to improve what &lt;tt&gt;--repair&lt;/tt&gt; can be done in this an other situations, so feel free to watch it and/or vote for it.&lt;/p&gt;

&lt;p&gt;Lastly, if you are able to reproduce these problems in a simple environment (e.g.: local nfs mount) and want to share your results with us that would be of great help, as it would help with &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-19815&quot; title=&quot;Improved mongod --repair option for WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-19815&quot;&gt;&lt;del&gt;SERVER-19815&lt;/del&gt;&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="1215828" author="ck-lee" created="Fri, 25 Mar 2016 20:45:18 +0000"  >&lt;p&gt;Can you help me recover the WiredTiger.wt and WiredTiger.turtle files because I&apos;m stucked now as I realized my last good backup is on the 9th of March &lt;img class=&quot;emoticon&quot; src=&quot;https://jira.mongodb.org/images/icons/emoticons/sad.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt; ? I&apos;m happy to try it myself if you can provide a guide with how you helped &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-23122&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;https://jira.mongodb.org/browse/SERVER-23122&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1215813" author="ck-lee" created="Fri, 25 Mar 2016 20:31:01 +0000"  >&lt;p&gt;The setup that corrupts the WiredTiger.wt is&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;A standalone mongodb instance. (Not a replica-set)&lt;/li&gt;
	&lt;li&gt;The mongodb daemon runs on a kubernetes pod using the official mongodb:3.2.4 docker image.&lt;/li&gt;
	&lt;li&gt;The files in &lt;tt&gt;/data/db&lt;/tt&gt; is persisted using kubernetes persistent volume (nfs mount). Hosted on a separate file server (coreos).&lt;/li&gt;
	&lt;li&gt;The pod can be moved to different nodes without causing issues &lt;em&gt;if&lt;/em&gt; the pod is shutdown properly.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;I had an outage on a node, which caused the pod not to shutdown properly. And when kubernetes trying to start the mongodb pod in a different node. It fails because the WiredTiget.wt is corrupted. Only files in &lt;tt&gt;/data/db&lt;/tt&gt; is persisted, so I don&apos;t have other log files. But I do have the &lt;tt&gt;diagnostic.data&lt;/tt&gt; folder. I hope that helps.&lt;/p&gt;

&lt;p&gt;In my case, I believe it is an operational problem which corrupted WiredTiger.wt file because mongod did not exit properly. But the issue is the database cannot be recovered by running &lt;tt&gt;--repair&lt;/tt&gt; flag.&lt;/p&gt;

&lt;p&gt;Would you like me to try and reproduce this to help determine if there is a bug that causes WiredTiger.wt on an unclean exit? I am keen to help to prevent this in the future.&lt;/p&gt;</comment>
                            <comment id="1215679" author="ramon.fernandez" created="Fri, 25 Mar 2016 18:48:07 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=ck-lee&quot; class=&quot;user-hover&quot; rel=&quot;ck-lee&quot;&gt;ck-lee&lt;/a&gt;, can you please provide some more details on the setup and, specifically, how was this node restarted? Being able to see the logs a few minutes before and after the restart should help us determine if this is a bug in the server or an operational problem.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="1215149" author="ck-lee" created="Fri, 25 Mar 2016 08:08:21 +0000"  >&lt;p&gt;I have attached my WiredTiger.wt and WiredTiger.turtle file. Can this be recovered? Thank you.&lt;/p&gt;</comment>
                            <comment id="1215146" author="ck-lee" created="Fri, 25 Mar 2016 08:03:54 +0000"  >&lt;p&gt;This issue is similar to &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-23122&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;https://jira.mongodb.org/browse/SERVER-23122&lt;/a&gt;&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="115018" name="WiredTiger.turtle" size="943" author="ck-lee" created="Fri, 25 Mar 2016 08:07:07 +0000"/>
                            <attachment id="115019" name="WiredTiger.wt" size="176128" author="ck-lee" created="Fri, 25 Mar 2016 08:07:07 +0000"/>
                            <attachment id="115114" name="metrics.zip" size="4816919" author="ck-lee" created="Fri, 25 Mar 2016 20:32:45 +0000"/>
                            <attachment id="115257" name="repair_attempt.tgz" size="9881" author="ramon.fernandez@mongodb.com" created="Mon, 28 Mar 2016 18:21:56 +0000"/>
                            <attachment id="115379" name="screendump.txt" size="32742" author="ck-lee" created="Tue, 29 Mar 2016 08:17:30 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>16.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 25 Mar 2016 18:48:07 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        6 years, 47 weeks, 6 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>nick.brewer</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            6 years, 47 weeks, 6 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10020"><![CDATA[Linux]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>alexander.gorrod@mongodb.com</customfieldvalue>
            <customfieldvalue>Bash</customfieldvalue>
            <customfieldvalue>ck-lee</customfieldvalue>
            <customfieldvalue>david.hows</customfieldvalue>
            <customfieldvalue>kelsey.schubert@mongodb.com</customfieldvalue>
            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrkchj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hsjrof:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hs5hav:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>