<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:53:13 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-39824] WiredTiger.wt: encountered an illegal file format or internal value</title>
                <link>https://jira.mongodb.org/browse/SERVER-39824</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Looks like this error: &lt;a href=&quot;http://example.com/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://jira.mongodb.org/browse/SERVER-26103&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;I am running a JuJu Controller that had 2 other host in a replication state. However, during a mishap the other two got wiped and upon rebooting and attempting to restore the database I am running in now that shows the following errors:&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;
&lt;blockquote&gt;&lt;p&gt;Feb 25 18:34:27 hqosjuju systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: Started juju state database.&lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: 2019-02-25T18:34:27.614+0000 W CONTROL &lt;span class=&quot;error&quot;&gt;&amp;#91;main&amp;#93;&lt;/span&gt; No SSL certificate validation can be performed since no CA file has been provided; please specify an sslCAFile parameter&lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; MongoDB starting : pid=1410 port=37017 dbpath=/var/lib/juju/db 64-bit host=hqosjuju&lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; db version v3.2.15&lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; git version: e11e3c1b9c9ce3f7b4a79493e16f5e4504e01140&lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; OpenSSL version: OpenSSL 1.0.2g 1 Mar 2016&lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; allocator: tcmalloc&lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; modules: none&lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; build environment:&lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; distarch: x86_64&lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; target_arch: x86_64&lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; options: { net: { ipv6: true, port: 37017, ssl: &lt;/p&gt;
&lt;div class=&quot;error&quot;&gt;&lt;span class=&quot;error&quot;&gt;Unknown macro: { PEMKeyFile}&lt;/span&gt; &lt;/div&gt;
&lt;p&gt; }, replication: { oplogSizeMB: 1024, replSet: &quot;juju&quot; }, security: { authorization: &quot;enabled&quot;, keyFile: &quot;/var/lib/juju/shared-secret&quot; }, storage: { dbPath: &quot;/var/lib/juju/db&quot;, engine: &quot;wiredTiger&quot;, journal: &lt;/p&gt;
&lt;div class=&quot;error&quot;&gt;&lt;span class=&quot;error&quot;&gt;Unknown macro: { enabled}&lt;/span&gt; &lt;/div&gt;
&lt;p&gt;, wiredTiger: { engineConfig: { cacheSizeGB: &lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; wiredtiger_open config: create,cache_size=1G,session_max=20000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),&lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; WiredTiger (0) &lt;span class=&quot;error&quot;&gt;&amp;#91;1551119667:660526&amp;#93;&lt;/span&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;1410:0x7f4db7054bc0&amp;#93;&lt;/span&gt;, &lt;a href=&quot;file:WiredTiger.wt&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:WiredTiger.wt&lt;/a&gt;, connection: read checksum error for 4096B block at offset 540672: block header checksum of 1881944125 doesn&apos;t match expected checksum of 28637535&lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; WiredTiger (0) &lt;span class=&quot;error&quot;&gt;&amp;#91;1551119667:660613&amp;#93;&lt;/span&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;1410:0x7f4db7054bc0&amp;#93;&lt;/span&gt;, &lt;a href=&quot;file:WiredTiger.wt&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:WiredTiger.wt&lt;/a&gt;, connection: WiredTiger.wt: encountered an illegal file format or internal value&lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; WiredTiger (-31804) &lt;span class=&quot;error&quot;&gt;&amp;#91;1551119667:660627&amp;#93;&lt;/span&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;1410:0x7f4db7054bc0&amp;#93;&lt;/span&gt;, &lt;a href=&quot;file:WiredTiger.wt&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:WiredTiger.wt&lt;/a&gt;, connection: the process must exit and restart: WT_PANIC: WiredTiger library panic&lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Fatal Assertion 28558&lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; &lt;/p&gt;

&lt;p&gt; ***aborting after fassert() failure&lt;br/&gt;
Feb 25 18:34:27 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1410&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Got signal: 6 (Aborted).&lt;/p&gt;

&lt;p&gt; 0x12a7701 0x12a6559 0x12a6e81 0x7f4db3ff5390 0x7f4db3c4f428 0x7f4db3c5102a 0x12209f2 0x1000efa 0x6f6234 0x6f6450 0x6f66a8 0x1356faf 0x13574fb 0x1353aed 0x13586c7 0x13721cb 0x13ab4d3 0x14357db 0x1435d1d 0x1435fdc 0x13b9bd1 0x14324f8 0x13f5c0e 0x13f5ceb 0x13a76e9 0xfe4fad 0xfdd474 0xeceb7e 0x73bd14 0x6f73a2 0x7f4db3c3a830 0x736e99&lt;br/&gt;
 ----- BEGIN BACKTRACE -----&lt;br/&gt;
 {&quot;backtrace&quot;:[&lt;/p&gt;
&lt;div class=&quot;error&quot;&gt;&lt;span class=&quot;error&quot;&gt;Unknown macro: {&amp;quot;b&amp;quot;}&lt;/span&gt; &lt;/div&gt;
&lt;p&gt;,{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;EA6559&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;EA6E81&quot;},{&quot;b&quot;:&quot;7F4DB3FE4000&quot;,&quot;o&quot;:&quot;11390&quot;},{&quot;b&quot;:&quot;7F4DB3C1A000&quot;,&quot;o&quot;:&quot;35428&quot;,&quot;s&quot;:&quot;gsignal&quot;},{&quot;b&quot;:&quot;7F4DB3C1A000&quot;,&quot;o&quot;:&quot;3702A&quot;,&quot;s&quot;:&quot;abort&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;E209F2&quot;,&quot;s&quot;:&quot;&lt;em&gt;ZN5mongo13fassertFailedEi&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;C00EFA&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;2F6234&quot;,&quot;s&quot;:&quot;&lt;/em&gt;_wt_eventv&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;2F6450&quot;,&quot;s&quot;&lt;br/&gt;
 mongod(_ZN5mongo15printStackTraceERSo+0x41) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x12a7701&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(+0xEA6559) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x12a6559&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(+0xEA6E81) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x12a6e81&amp;#93;&lt;/span&gt;&lt;br/&gt;
 libpthread.so.0(+0x11390) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7f4db3ff5390&amp;#93;&lt;/span&gt;&lt;br/&gt;
 libc.so.6(gsignal+0x38) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7f4db3c4f428&amp;#93;&lt;/span&gt;&lt;br/&gt;
 libc.so.6(abort+0x16A) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7f4db3c5102a&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo13fassertFailedEi+0xA2) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x12209f2&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(+0xC00EFA) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1000efa&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(__wt_eventv+0x3D7) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x6f6234&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(__wt_err+0x9D) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x6f6450&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(__wt_panic+0x24) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x6f66a8&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(__wt_block_extlist_read+0x8F) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1356faf&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(__wt_block_extlist_read_avail+0x2B) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x13574fb&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(__wt_block_checkpoint_load+0x26D) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1353aed&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(+0xF586C7) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x13586c7&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(__wt_btree_open+0xB3B) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x13721cb&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(__wt_conn_btree_open+0x163) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x13ab4d3&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(__wt_session_get_btree+0xFB) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x14357db&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(__wt_session_get_btree+0x63D) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1435d1d&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(__wt_session_get_btree_ckpt+0x14C) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1435fdc&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(__wt_curfile_open+0x161) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x13b9bd1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(+0x10324F8) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x14324f8&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(__wt_metadata_cursor_open+0x6E) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x13f5c0e&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(__wt_metadata_cursor+0x4B) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x13f5ceb&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(wiredtiger_open+0x1659) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x13a76e9&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(&lt;em&gt;ZN5mongo18WiredTigerKVEngineC1ERKNSt7&lt;/em&gt;_cxx1112basic_stringIcSt11char_traitsIcESaIcEEES8_S8_mbbb+0xA6D) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xfe4fad&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(+0xBDD474) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xfdd474&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo20ServiceContextMongoD29initializeGlobalStorageEngineEv+0x3FE) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xeceb7e&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(+0x33BD14) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x73bd14&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(main+0x732) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x6f73a2&amp;#93;&lt;/span&gt;&lt;br/&gt;
 libc.so.6(__libc_start_main+0xF0) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7f4db3c3a830&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_start+0x29) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x736e99&amp;#93;&lt;/span&gt;&lt;br/&gt;
 ----- END BACKTRACE -----&lt;br/&gt;
Feb 25 18:34:27 hqosjuju systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: juju-db.service: Main process exited, code=dumped, status=6/ABRT&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;It was running on a VM that was restarted incorrectly and caused some disk corruption, which has been since resolved. I currently have a backup of the corrupt VM, a Backup of the database which was ran before the VM was corrupted, and a semi-working restored database on a new VM.&lt;/p&gt;

&lt;p&gt;I was able to get the database working on a new VM with the issue in juju (not mongo related) that the 509x cert is incorrect so I cannot use that server. I want to restore to the original VM (with the right certs and disk corruption issues resolved) but cannot start mongod without these issues above. It looks like the WiredTiger.wt file is corrupt. I have seen multiple forum posts and Jira issues where you guys have repaired the issue but provided no insight in to how. So...I am posting the files here.&lt;/p&gt;

&lt;p&gt;If there is a way to restore a database without starting a database, I would love to see documentation on that, as thus far I can find none. I have both BSON files from a dump as well as a restore with all of the .wt files (a ton of them) I provided all of the WiredTiger files as most previous posts have requested.&lt;/p&gt;</description>
                <environment>QEMU/KVM Virtual Machine&lt;br/&gt;
Ubuntu 16.04.5</environment>
        <key id="704146">SERVER-39824</key>
            <summary>WiredTiger.wt: encountered an illegal file format or internal value</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="eric.sedor@mongodb.com">Eric Sedor</assignee>
                                    <reporter username="matthew.williams">Matthew Williams</reporter>
                        <labels>
                            <label>WiredTiger.wt</label>
                            <label>corrupt</label>
                            <label>wt-repair-success</label>
                    </labels>
                <created>Mon, 25 Feb 2019 19:45:57 +0000</created>
                <updated>Thu, 14 Mar 2019 14:23:24 +0000</updated>
                            <resolved>Thu, 14 Mar 2019 14:23:24 +0000</resolved>
                                    <version>3.2.15</version>
                                                                        <votes>0</votes>
                                    <watches>6</watches>
                                                                                                                <comments>
                            <comment id="2180766" author="eric.sedor" created="Thu, 14 Mar 2019 14:23:13 +0000"  >&lt;p&gt;Glad to hear &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=matthew.williams&quot; class=&quot;user-hover&quot; rel=&quot;matthew.williams&quot;&gt;matthew.williams&lt;/a&gt;! To prevent this type of problem in the future please take note of the following guidelines to help mitigate any issues related to unreliable storage layers or server failures. &lt;/p&gt;

&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;Make sure your &lt;a href=&quot;https://docs.mongodb.com/manual/administration/production-notes/#disk-and-storage-systems&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;underlying storage is configured in an optimal way&lt;/a&gt;.&lt;/li&gt;
	&lt;li&gt;Schedule and perform regular checks of the integrity of your filesystems and disks.&lt;/li&gt;
	&lt;li&gt;Make sure to update MongoDB to the &lt;a href=&quot;https://docs.mongodb.com/manual/release-notes/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;most recent version&lt;/a&gt;.&lt;/li&gt;
	&lt;li&gt;Never manipulate the underlying database files in any way while &lt;tt&gt;mongod&lt;/tt&gt; is running.&lt;/li&gt;
	&lt;li&gt;Always keep up to date &lt;a href=&quot;https://docs.mongodb.com/manual/core/backups/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;backups of your databases&lt;/a&gt; and verify that you have a process in place to restore them.&lt;/li&gt;
	&lt;li&gt;Use a &lt;a href=&quot;https://docs.mongodb.com/manual/replication/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;replica set&lt;/a&gt; for improved reliability.&lt;/li&gt;
&lt;/ul&gt;
</comment>
                            <comment id="2180251" author="matthew.williams" created="Wed, 13 Mar 2019 21:08:36 +0000"  >&lt;p&gt;This resolved the issue and it has survived a number of tests on it. I have other issues now, but those are mostly related to juju and its database management and not directly related to mongo. Thank you all for your help.&lt;/p&gt;</comment>
                            <comment id="2172746" author="eric.sedor" created="Wed, 6 Mar 2019 16:14:43 +0000"  >&lt;p&gt;Thanks for keeping us updated; good to hear so far!&lt;/p&gt;</comment>
                            <comment id="2172741" author="matthew.williams" created="Wed, 6 Mar 2019 16:11:18 +0000"  >&lt;p&gt;This appears to have successfully repaired the live database, I am running it through some tests and attempting to remove replication before determining if this was 100% successful.&lt;/p&gt;</comment>
                            <comment id="2167925" author="matthew.williams" created="Fri, 1 Mar 2019 20:43:17 +0000"  >&lt;p&gt;This appears to have functioned on the test server. I am testing it on the live server now.&#160;&lt;/p&gt;</comment>
                            <comment id="2165367" author="eric.sedor" created="Wed, 27 Feb 2019 21:37:37 +0000"  >&lt;p&gt;Sorry to hear, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=matthew.williams&quot; class=&quot;user-hover&quot; rel=&quot;matthew.williams&quot;&gt;matthew.williams&lt;/a&gt;. It&apos;s indeed a good move working off of a copy.&lt;/p&gt;

&lt;p&gt;Our next recommendation would be to attempt a similar mongod --repair but using mongod version 4.0.5+. Can you let us know if that helps?&lt;/p&gt;</comment>
                            <comment id="2165350" author="matthew.williams" created="Wed, 27 Feb 2019 21:26:27 +0000"  >&lt;p&gt;Ok, after a reboot and a few other attempts to repair/restore the database. I am down to this:&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;
&lt;blockquote&gt;&lt;p&gt;Feb 27 20:29:48 hqosjuju systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: Started juju state database.&lt;br/&gt;
&amp;#8211; Subject: Unit juju-db.service has finished start-up&lt;br/&gt;
&amp;#8211; Defined-By: systemd&lt;br/&gt;
&amp;#8211; Support: &lt;a href=&quot;http://lists.freedesktop.org/mailman/listinfo/systemd-devel&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://lists.freedesktop.org/mailman/listinfo/systemd-devel&lt;/a&gt;&lt;br/&gt;
&amp;#8211; &lt;br/&gt;
&amp;#8211; Unit juju-db.service has finished starting up.&lt;br/&gt;
&amp;#8211; &lt;br/&gt;
&amp;#8211; The start-up result is done.&lt;br/&gt;
Feb 27 20:29:48 hqosjuju mongod&lt;span class=&quot;error&quot;&gt;&amp;#91;1527&amp;#93;&lt;/span&gt;: 2019-02-27T20:29:48.947+0000 W CONTROL &lt;span class=&quot;error&quot;&gt;&amp;#91;main&amp;#93;&lt;/span&gt; No SSL certificate validation can be performed since no CA file has been provided; please specify an sslCAFile parameter&lt;br/&gt;
Feb 27 20:29:48 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1527&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; MongoDB starting : pid=1527 port=37017 dbpath=/var/lib/juju/db 64-bit host=hqosjuju&lt;br/&gt;
Feb 27 20:29:48 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1527&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; db version v3.2.15&lt;br/&gt;
Feb 27 20:29:48 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1527&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; git version: e11e3c1b9c9ce3f7b4a79493e16f5e4504e01140&lt;br/&gt;
Feb 27 20:29:48 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1527&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; OpenSSL version: OpenSSL 1.0.2g 1 Mar 2016&lt;br/&gt;
Feb 27 20:29:48 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1527&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; allocator: tcmalloc&lt;br/&gt;
Feb 27 20:29:48 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1527&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; modules: none&lt;br/&gt;
Feb 27 20:29:48 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1527&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; build environment:&lt;br/&gt;
Feb 27 20:29:48 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1527&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; distarch: x86_64&lt;br/&gt;
Feb 27 20:29:48 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1527&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; target_arch: x86_64&lt;br/&gt;
Feb 27 20:29:48 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1527&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; options: { net: { ipv6: true, port: 37017, ssl: &lt;/p&gt;
&lt;div class=&quot;error&quot;&gt;&lt;span class=&quot;error&quot;&gt;Unknown macro: { PEMKeyFile}&lt;/span&gt; &lt;/div&gt;
&lt;p&gt; }, replication: { oplogSizeMB: 5&lt;br/&gt;
Feb 27 20:29:48 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1527&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Detected unclean shutdown - /var/lib/juju/db/mongod.lock is not empty.&lt;br/&gt;
Feb 27 20:29:48 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1527&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Recovering data from the last clean checkpoint.&lt;br/&gt;
Feb 27 20:29:48 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1527&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; wiredtiger_open config: create,cache_size=1G,session_max=20000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path&lt;br/&gt;
Feb 27 20:29:49 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1527&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Fatal Assertion 34433&lt;br/&gt;
Feb 27 20:29:49 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1527&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; &lt;/p&gt;

&lt;p&gt; ***aborting after fassert() failure&lt;br/&gt;
Feb 27 20:29:49 hqosjuju systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: juju-db.service: Main process exited, code=exited, status=14/n/a&lt;br/&gt;
Feb 27 20:29:49 hqosjuju systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: juju-db.service: Unit entered failed state.&lt;br/&gt;
Feb 27 20:29:49 hqosjuju systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: juju-db.service: Failed with result &apos;exit-code&apos;.&lt;br/&gt;
Feb 27 20:29:49 hqosjuju systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: juju-db.service: Service hold-off time over, scheduling restart.&lt;br/&gt;
Feb 27 20:29:49 hqosjuju systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: Stopped juju state database.&lt;br/&gt;
&amp;#8211; Subject: Unit juju-db.service has finished shutting down&lt;br/&gt;
&amp;#8211; Defined-By: systemd&lt;br/&gt;
&amp;#8211; Support: &lt;a href=&quot;http://lists.freedesktop.org/mailman/listinfo/systemd-devel&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://lists.freedesktop.org/mailman/listinfo/systemd-devel&lt;/a&gt;&lt;br/&gt;
&amp;#8211; &lt;br/&gt;
&amp;#8211; Unit juju-db.service has finished shutting down.&lt;br/&gt;
Feb 27 20:29:49 hqosjuju systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: juju-db.service: Start request repeated too quickly.&lt;br/&gt;
Feb 27 20:29:49 hqosjuju systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: Failed to start juju state database.&lt;br/&gt;
&amp;#8211; Subject: Unit juju-db.service has failed&lt;br/&gt;
&amp;#8211; Defined-By: systemd&lt;br/&gt;
&amp;#8211; Support: &lt;a href=&quot;http://lists.freedesktop.org/mailman/listinfo/systemd-devel&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://lists.freedesktop.org/mailman/listinfo/systemd-devel&lt;/a&gt;&lt;br/&gt;
&amp;#8211; &lt;br/&gt;
&amp;#8211; Unit juju-db.service has failed.&lt;/p&gt;

&lt;/blockquote&gt;
&lt;p&gt;I will await further isntruction/repair attempts.&lt;/p&gt;</comment>
                            <comment id="2165315" author="matthew.williams" created="Wed, 27 Feb 2019 21:10:15 +0000"  >&lt;p&gt;This has led to this error during the repair:&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;
&lt;blockquote&gt;&lt;p&gt;root@server:/home/server# /usr/lib/juju/mongo3.2/bin/mongod --repair --dbpath &apos;/var/lib/juju/db&apos;&lt;br/&gt;
 2019-02-27T20:25:02.925+0000 I CONTROL &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; MongoDB starting : pid=1611 port=27017 dbpath=/var/lib/juju/db 64-bit host=hqosjuju&lt;br/&gt;
 2019-02-27T20:25:02.925+0000 I CONTROL &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; db version v3.2.15&lt;br/&gt;
 2019-02-27T20:25:02.925+0000 I CONTROL &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; git version: e11e3c1b9c9ce3f7b4a79493e16f5e4504e01140&lt;br/&gt;
 2019-02-27T20:25:02.925+0000 I CONTROL &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; OpenSSL version: OpenSSL 1.0.2g 1 Mar 2016&lt;br/&gt;
 2019-02-27T20:25:02.925+0000 I CONTROL &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; allocator: tcmalloc&lt;br/&gt;
 2019-02-27T20:25:02.925+0000 I CONTROL &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; modules: none&lt;br/&gt;
 2019-02-27T20:25:02.925+0000 I CONTROL &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; build environment:&lt;br/&gt;
 2019-02-27T20:25:02.925+0000 I CONTROL &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; distarch: x86_64&lt;br/&gt;
 2019-02-27T20:25:02.925+0000 I CONTROL &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; target_arch: x86_64&lt;br/&gt;
 2019-02-27T20:25:02.925+0000 I CONTROL &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; options: &lt;/p&gt;
&lt;div class=&quot;error&quot;&gt;&lt;span class=&quot;error&quot;&gt;Unknown macro: { repair}&lt;/span&gt; &lt;/div&gt;
&lt;p&gt;}&lt;br/&gt;
 2019-02-27T20:25:02.945+0000 I - &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Detected data files in /var/lib/juju/db created by the &apos;wiredTiger&apos; storage engine, so setting the active storage engine to &apos;wiredTiger&apos;.&lt;br/&gt;
 2019-02-27T20:25:02.945+0000 W - &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Detected unclean shutdown - /var/lib/juju/db/mongod.lock is not empty.&lt;br/&gt;
 2019-02-27T20:25:02.945+0000 W STORAGE &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Recovering data from the last clean checkpoint.&lt;br/&gt;
 2019-02-27T20:25:02.945+0000 I STORAGE &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Detected WT journal files. Running recovery from last checkpoint.&lt;br/&gt;
 2019-02-27T20:25:02.945+0000 I STORAGE &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; journal to nojournal transition config: create,cache_size=1G,session_max=20000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),&lt;br/&gt;
 2019-02-27T20:25:03.018+0000 I STORAGE &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; wiredtiger_open config: create,cache_size=1G,session_max=20000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),,log=(enabled=false),&lt;br/&gt;
 2019-02-27T20:25:03.058+0000 I STORAGE &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Repairing size cache&lt;br/&gt;
 2019-02-27T20:25:03.059+0000 I STORAGE &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Verify succeeded on uri table:sizeStorer. Not salvaging.&lt;br/&gt;
 2019-02-27T20:25:03.060+0000 I STORAGE &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Repairing catalog metadata&lt;br/&gt;
 2019-02-27T20:25:03.062+0000 I STORAGE &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Verify succeeded on uri table:_mdb_catalog. Not salvaging.&lt;br/&gt;
 2019-02-27T20:25:03.068+0000 I CONTROL &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; ** WARNING: You are running this process as the root user, which is not recommended.&lt;br/&gt;
 2019-02-27T20:25:03.068+0000 I CONTROL &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; &lt;br/&gt;
 2019-02-27T20:25:03.068+0000 I STORAGE &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; repairDatabase admin&lt;br/&gt;
 2019-02-27T20:25:03.069+0000 I STORAGE &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Repairing collection admin.system.users&lt;br/&gt;
 2019-02-27T20:25:03.069+0000 I STORAGE &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Verify failed on uri table:collection-13-8833743074834640358. Running a salvage operation.&lt;br/&gt;
 2019-02-27T20:25:03.069+0000 I - &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Invariant failure rs.get() src/mongo/db/catalog/database.cpp 190&lt;br/&gt;
 2019-02-27T20:25:03.069+0000 I - &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;***aborting after invariant() failure&lt;/p&gt;

&lt;p&gt;2019-02-27T20:25:03.072+0000 F - &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Got signal: 6 (Aborted).&lt;/p&gt;

&lt;p&gt;0x12a7701 0x12a6559 0x12a6e81 0x7f02ae3fa390 0x7f02ae054428 0x7f02ae05602a 0x12208c4 0x8b6680 0x8bdb30 0x8c09b0 0xd1f0b1 0x739325 0x73c4c0 0x6f73a2 0x7f02ae03f830 0x736e99&lt;br/&gt;
 ----- BEGIN BACKTRACE -----&lt;br/&gt;
Unknown macro: {&quot;backtrace&quot;}&lt;br/&gt;
,{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;EA6559&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;EA6E81&quot;},{&quot;b&quot;:&quot;7F02AE3E9000&quot;,&quot;o&quot;:&quot;11390&quot;},{&quot;b&quot;:&quot;7F02AE01F000&quot;,&quot;o&quot;:&quot;35428&quot;,&quot;s&quot;:&quot;gsignal&quot;},{&quot;b&quot;:&quot;7F02AE01F000&quot;,&quot;o&quot;:&quot;3702A&quot;,&quot;s&quot;:&quot;abort&quot;},&lt;/p&gt;
&lt;div class=&quot;error&quot;&gt;&lt;span class=&quot;error&quot;&gt;Unknown macro: {&amp;quot;b&amp;quot;}&lt;/span&gt; &lt;/div&gt;
&lt;p&gt;,{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;4B6680&quot;,&quot;s&quot;:&quot;_ZN5mongo8Database30_getOrCreateCollectionInstanceEPNS_16OperationContextENS_10StringDataE&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;4BDB30&quot;,&quot;s&quot;:&quot;_ZN5mongo8DatabaseC1EPNS_16OperationContextENS_10StringDataEPNS_20DatabaseCatalogEntryE&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;4C09B0&quot;,&quot;s&quot;:&quot;_ZN5mongo14DatabaseHolder6openDbEPNS_16OperationContextENS_10StringDataEPb&quot;},&lt;/p&gt;
&lt;div class=&quot;error&quot;&gt;&lt;span class=&quot;error&quot;&gt;Unknown macro: {&amp;quot;b&amp;quot;}&lt;/span&gt; &lt;/div&gt;
&lt;p&gt;,{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;339325&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;33C4C0&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;2F73A2&quot;,&quot;s&quot;:&quot;main&quot;},&lt;/p&gt;
&lt;div class=&quot;error&quot;&gt;&lt;span class=&quot;error&quot;&gt;Unknown macro: {&amp;quot;b&amp;quot;}&lt;/span&gt; &lt;/div&gt;
&lt;p&gt;,{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;336E99&quot;,&quot;s&quot;:&quot;_start&quot;}],&quot;processInfo&quot;:&lt;/p&gt;
&lt;div class=&quot;error&quot;&gt;&lt;span class=&quot;error&quot;&gt;Unknown macro: { &amp;quot;mongodbVersion&amp;quot; }&lt;/span&gt; &lt;/div&gt;
&lt;p&gt;, &quot;somap&quot; : [ { &quot;elfType&quot; : 2, &quot;b&quot; : &quot;400000&quot;, &quot;buildId&quot; : &quot;154504DC4E334AF6C6D92A625384E0136F3E5837&quot; }, { &quot;b&quot; : &quot;7FFF8065D000&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;EFD9CD6D855DEE3F29B3F5CA8D2C6C1C2B7D7CAB&quot; }, { &quot;b&quot; : &quot;7F02B0FD9000&quot;, &quot;path&quot; : &quot;/usr/lib/libtcmalloc.so.4&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;C376C112685221C43033ED32DDD1E138B658483B&quot; }, { &quot;b&quot; : &quot;7F02B0DBF000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libz.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;8D9BD4CE26E45EF16075C67D5F5EEAFD8B562832&quot; }, { &quot;b&quot; : &quot;7F02B0BB7000&quot;, &quot;path&quot; : &quot;/usr/lib/x86_64-linux-gnu/libsnappy.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;CE5C34DFF6A98121F82EED45478616455801DB1D&quot; }, { &quot;b&quot; : &quot;7F02B0939000&quot;, &quot;path&quot; : &quot;/usr/lib/x86_64-linux-gnu/libboost_program_options.so.1.58.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;9F70D8EB5739EEF251E6DE6DED289027DA61844E&quot; }, { &quot;b&quot; : &quot;7F02B0721000&quot;, &quot;path&quot; : &quot;/usr/lib/x86_64-linux-gnu/libboost_filesystem.so.1.58.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;FC0239AC1E59EB10991A2423B48986ADB010F9D3&quot; }, { &quot;b&quot; : &quot;7F02B04FB000&quot;, &quot;path&quot; : &quot;/usr/lib/x86_64-linux-gnu/libboost_thread.so.1.58.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;93798D047035F12B0AA8DE4DDF45DA8636519986&quot; }, { &quot;b&quot; : &quot;7F02B02F7000&quot;, &quot;path&quot; : &quot;/usr/lib/x86_64-linux-gnu/libboost_system.so.1.58.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;3EBF263E88DAE32EE640C9C599A9D7DD59F3DF28&quot; }, { &quot;b&quot; : &quot;7F02B00EF000&quot;, &quot;path&quot; : &quot;/usr/lib/x86_64-linux-gnu/libboost_chrono.so.1.58.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;34C98695297484FCDE0AD442BEB8412F1B787CE7&quot; }, { &quot;b&quot; : &quot;7F02AFDE7000&quot;, &quot;path&quot; : &quot;/usr/lib/x86_64-linux-gnu/libboost_regex.so.1.58.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;FF52EBC3EA55DD649B9A6D65D8AC7D73CF2B99D7&quot; }, { &quot;b&quot; : &quot;7F02AFBDE000&quot;, &quot;path&quot; : &quot;/usr/lib/x86_64-linux-gnu/libpcrecpp.so.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;96888DAFDBE35DEDCAE309DB9C0943C2C2271EB1&quot; }, { &quot;b&quot; : &quot;7F02AF961000&quot;, &quot;path&quot; : &quot;/usr/lib/x86_64-linux-gnu/libyaml-cpp.so.0.5&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;D163879ED83585A5B2B58D41CDCB78ABBBD351C5&quot; }, { &quot;b&quot; : &quot;7F02AF6F8000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libssl.so.1.0.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;473092A9AF373FB0CAB555F9A003BC67F47756B6&quot; }, { &quot;b&quot; : &quot;7F02AF2B3000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libcrypto.so.1.0.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;8942CA58A3B910E883CC31E04A23DBD09729B4B0&quot; }, { &quot;b&quot; : &quot;7F02AF0AB000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/librt.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;69143E8B39040C964D3958490535322675F15DD3&quot; }, { &quot;b&quot; : &quot;7F02AEEA7000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libdl.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;37BFC3D8F7E3B022DAC7943B1A5FACD40CEBF0AD&quot; }, { &quot;b&quot; : &quot;7F02AEB25000&quot;, &quot;path&quot; : &quot;/usr/lib/x86_64-linux-gnu/libstdc++.so.6&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;C5CA582A8E1EE50F59CAAAF84A1464C8CB3C2F3D&quot; }, { &quot;b&quot; : &quot;7F02AE81C000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libm.so.6&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;BAD67A84E56E73D031AE507261DA066B35949D34&quot; }, { &quot;b&quot; : &quot;7F02AE606000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libgcc_s.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;68220AE2C65D65C1B6AAA12FA6765A6EC2F5F434&quot; }, { &quot;b&quot; : &quot;7F02AE3E9000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libpthread.so.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;B17C21299099640A6D863E423D99265824E7BB16&quot; }, { &quot;b&quot; : &quot;7F02AE01F000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libc.so.6&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;1CA54A6E0D76188105B12E49FE6B8019BF08803A&quot; }, { &quot;b&quot; : &quot;7F02B124A000&quot;, &quot;path&quot; : &quot;/lib64/ld-linux-x86-64.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;C0ADBAD6F9A33944F2B3567C078EC472A1DAE98E&quot; }, { &quot;b&quot; : &quot;7F02ADE04000&quot;, &quot;path&quot; : &quot;/usr/lib/x86_64-linux-gnu/libunwind.so.8&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;29F63E3B24F95E955C76E58354DB3294944105A1&quot; }, { &quot;b&quot; : &quot;7F02AD9A2000&quot;, &quot;path&quot; : &quot;/usr/lib/x86_64-linux-gnu/libicui18n.so.55&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;F5BE69B92B7426F17A61E1A1308115A9FE73C51D&quot; }, { &quot;b&quot; : &quot;7F02AD60E000&quot;, &quot;path&quot; : &quot;/usr/lib/x86_64-linux-gnu/libicuuc.so.55&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;463D8B610702D64AE0803C7DFCAA02CFB4C6477B&quot; }, { &quot;b&quot; : &quot;7F02AD39E000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libpcre.so.3&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;390B2228E9A1071BB0BE285D77B6669CB37CE628&quot; }, { &quot;b&quot; : &quot;7F02AD17C000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/liblzma.so.5&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;15AED4855920E5A0FB8791B683EB88C7E1199260&quot; }, { &quot;b&quot; : &quot;7F02AB6C5000&quot;, &quot;path&quot; : &quot;/usr/lib/x86_64-linux-gnu/libicudata.so.55&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;2CC92B3EC41116DD818F3DAF7DA7CBB410183DD6&quot; } ] }}&lt;br/&gt;
 mongod(_ZN5mongo15printStackTraceERSo+0x41) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x12a7701&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(+0xEA6559) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x12a6559&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(+0xEA6E81) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x12a6e81&amp;#93;&lt;/span&gt;&lt;br/&gt;
 libpthread.so.0(+0x11390) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7f02ae3fa390&amp;#93;&lt;/span&gt;&lt;br/&gt;
 libc.so.6(gsignal+0x38) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7f02ae054428&amp;#93;&lt;/span&gt;&lt;br/&gt;
 libc.so.6(abort+0x16A) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7f02ae05602a&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo15invariantFailedEPKcS1_j+0x114) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x12208c4&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo8Database30_getOrCreateCollectionInstanceEPNS_16OperationContextENS_10StringDataE+0xE0) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x8b6680&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo8DatabaseC1EPNS_16OperationContextENS_10StringDataEPNS_20DatabaseCatalogEntryE+0x480) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x8bdb30&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo14DatabaseHolder6openDbEPNS_16OperationContextENS_10StringDataEPb+0xAC0) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x8c09b0&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(&lt;em&gt;ZN5mongo14repairDatabaseEPNS_16OperationContextEPNS_13StorageEngineERKNSt7&lt;/em&gt;_cxx1112basic_stringIcSt11char_traitsIcESaIcEEEbb+0xDC1) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xd1f0b1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(+0x339325) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x739325&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(+0x33C4C0) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x73c4c0&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(main+0x732) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x6f73a2&amp;#93;&lt;/span&gt;&lt;br/&gt;
 libc.so.6(__libc_start_main+0xF0) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7f02ae03f830&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_start+0x29) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x736e99&amp;#93;&lt;/span&gt;&lt;br/&gt;
 ----- END BACKTRACE -----&lt;br/&gt;
 Aborted (core dumped)&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;Mind you, I am doing this all on a clone of the original so I wont ruin the original during this. If this fails, I need to know how to restore a copy of the DB to the same host. I have a bson/json dump from before this issue happened, however, I cannot run a restore because the database wont start up..not sure how to handle that. I have not run into any documentation or forum posts that list how to do that, which is confusing since I believe that its likely an issue others have had.&lt;/p&gt;</comment>
                            <comment id="2165198" author="matthew.williams" created="Wed, 27 Feb 2019 20:16:50 +0000"  >&lt;blockquote&gt;&lt;p&gt;&amp;#8211; Unit juju-db.service has finished starting up.&lt;br/&gt;
&amp;#8211; &lt;br/&gt;
&amp;#8211; The start-up result is done.&lt;br/&gt;
Feb 27 19:31:24 hqosjuju mongod&lt;span class=&quot;error&quot;&gt;&amp;#91;1624&amp;#93;&lt;/span&gt;: 2019-02-27T19:31:24.981+0000 W CONTROL &lt;span class=&quot;error&quot;&gt;&amp;#91;main&amp;#93;&lt;/span&gt; No SSL certificate validation can be performed since no CA file has been provided; please specify an sslCAFile parameter&lt;br/&gt;
Feb 27 19:31:25 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1624&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; MongoDB starting : pid=1624 port=37017 dbpath=/var/lib/juju/db 64-bit host=hqosjuju&lt;br/&gt;
Feb 27 19:31:25 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1624&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; db version v3.2.15&lt;br/&gt;
Feb 27 19:31:25 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1624&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; git version: e11e3c1b9c9ce3f7b4a79493e16f5e4504e01140&lt;br/&gt;
Feb 27 19:31:25 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1624&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; OpenSSL version: OpenSSL 1.0.2g 1 Mar 2016&lt;br/&gt;
Feb 27 19:31:25 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1624&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; allocator: tcmalloc&lt;br/&gt;
Feb 27 19:31:25 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1624&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; modules: none&lt;br/&gt;
Feb 27 19:31:25 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1624&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; build environment:&lt;br/&gt;
Feb 27 19:31:25 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1624&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; distarch: x86_64&lt;br/&gt;
Feb 27 19:31:25 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1624&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; target_arch: x86_64&lt;br/&gt;
Feb 27 19:31:25 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1624&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; options: { net: { ipv6: true, port: 37017, ssl: &lt;/p&gt;
&lt;div class=&quot;error&quot;&gt;&lt;span class=&quot;error&quot;&gt;Unknown macro: { PEMKeyFile}&lt;/span&gt; &lt;/div&gt;
&lt;p&gt; }, replication: { oplogSizeMB: 1&lt;br/&gt;
Feb 27 19:31:25 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1624&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; wiredtiger_open config: create,cache_size=1G,session_max=20000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path&lt;br/&gt;
Feb 27 19:31:25 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1624&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; WiredTiger (-31802) &lt;span class=&quot;error&quot;&gt;&amp;#91;1551295885:228871&amp;#93;&lt;/span&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;1624:0x7fd915cdebc0&amp;#93;&lt;/span&gt;, &lt;a href=&quot;file:sizeStorer.wt&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:sizeStorer.wt&lt;/a&gt;, WT_SESSION.open_cursor: unable to read root page from &lt;a href=&quot;file:sizeStorer.wt:&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;file:sizeStorer.wt:&lt;/a&gt; WT_ERROR: non-spe&lt;br/&gt;
Feb 27 19:31:25 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1624&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Invariant failure: ret resulted in status UnknownError: -31802: WT_ERROR: non-specific WiredTiger error at src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.cpp 67&lt;br/&gt;
Feb 27 19:31:25 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1624&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; &lt;br/&gt;
 0x12a7701 0x123d053 0x1222c52 0xffc703 0xfe518a 0xfdd474 0xeceb7e 0x73bd14 0x6f73a2 0x7fd9128c4830 0x736e99&lt;br/&gt;
 ----- BEGIN BACKTRACE -----&lt;br/&gt;
 {&quot;backtrace&quot;:[&lt;/p&gt;
&lt;div class=&quot;error&quot;&gt;&lt;span class=&quot;error&quot;&gt;Unknown macro: {&amp;quot;b&amp;quot;}&lt;/span&gt; &lt;/div&gt;
&lt;p&gt;,{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;E3D053&quot;,&quot;s&quot;:&quot;_ZN5mongo10logContextEPKc&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;E22C52&quot;,&quot;s&quot;:&quot;_ZN5mongo17invaria&lt;br/&gt;
 mongod(_ZN5mongo15printStackTraceERSo+0x41) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x12a7701&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo10logContextEPKc+0x183) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x123d053&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0x102) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x1222c52&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(&lt;em&gt;ZN5mongo20WiredTigerSizeStorerC2EP15&lt;/em&gt;&lt;em&gt;wt_connectionRKNSt7&lt;/em&gt;_cxx1112basic_stringIcSt11char_traitsIcESaIcEEE+0x1D3) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xffc703&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(&lt;em&gt;ZN5mongo18WiredTigerKVEngineC1ERKNSt7&lt;/em&gt;_cxx1112basic_stringIcSt11char_traitsIcESaIcEEES8_S8_mbbb+0xC4A) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xfe518a&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(+0xBDD474) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xfdd474&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo20ServiceContextMongoD29initializeGlobalStorageEngineEv+0x3FE) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xeceb7e&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(+0x33BD14) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x73bd14&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(main+0x732) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x6f73a2&amp;#93;&lt;/span&gt;&lt;br/&gt;
 libc.so.6(__libc_start_main+0xF0) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fd9128c4830&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_start+0x29) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x736e99&amp;#93;&lt;/span&gt;&lt;br/&gt;
 ----- END BACKTRACE -----&lt;br/&gt;
Feb 27 19:31:25 hqosjuju mongod.37017&lt;span class=&quot;error&quot;&gt;&amp;#91;1624&amp;#93;&lt;/span&gt;: &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; &lt;/p&gt;

&lt;p&gt; ***aborting after invariant() failure&lt;br/&gt;
Feb 27 19:31:25 hqosjuju systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: juju-db.service: Main process exited, code=exited, status=14/n/a&lt;br/&gt;
Feb 27 19:31:25 hqosjuju systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: juju-db.service: Unit entered failed state.&lt;br/&gt;
Feb 27 19:31:25 hqosjuju systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: juju-db.service: Failed with result &apos;exit-code&apos;.&lt;br/&gt;
Feb 27 19:31:25 hqosjuju systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: juju-db.service: Service hold-off time over, scheduling restart.&lt;br/&gt;
Feb 27 19:31:25 hqosjuju systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: Stopped juju state database.&lt;br/&gt;
&amp;#8211; Subject: Unit juju-db.service has finished shutting down&lt;br/&gt;
&amp;#8211; Defined-By: systemd&lt;br/&gt;
&amp;#8211; Support: &lt;a href=&quot;http://lists.freedesktop.org/mailman/listinfo/systemd-devel&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://lists.freedesktop.org/mailman/listinfo/systemd-devel&lt;/a&gt;&lt;br/&gt;
&amp;#8211; &lt;br/&gt;
&amp;#8211; Unit juju-db.service has finished shutting down.&lt;br/&gt;
Feb 27 19:31:25 hqosjuju systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: juju-db.service: Start request repeated too quickly.&lt;br/&gt;
Feb 27 19:31:25 hqosjuju systemd&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;: Failed to start juju state database.&lt;br/&gt;
&amp;#8211; Subject: Unit juju-db.service has failed&lt;br/&gt;
&amp;#8211; Defined-By: systemd&lt;br/&gt;
&amp;#8211; Support: &lt;a href=&quot;http://lists.freedesktop.org/mailman/listinfo/systemd-devel&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://lists.freedesktop.org/mailman/listinfo/systemd-devel&lt;/a&gt;&lt;br/&gt;
&amp;#8211; &lt;br/&gt;
&amp;#8211; Unit juju-db.service has failed.&lt;br/&gt;
&amp;#8211; &lt;br/&gt;
&amp;#8211; The result is failed.&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;Getting this error at the moment. Going to try a mongod --repair to resolve it and research further.&lt;/p&gt;</comment>
                            <comment id="2164728" author="matthew.williams" created="Wed, 27 Feb 2019 15:29:26 +0000"  >&lt;p&gt;Thank you, testing now.&lt;/p&gt;</comment>
                            <comment id="2163795" author="eric.sedor" created="Wed, 27 Feb 2019 00:13:00 +0000"  >&lt;p&gt;Thanks for the detailed report &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=matthew.williams&quot; class=&quot;user-hover&quot; rel=&quot;matthew.williams&quot;&gt;matthew.williams&lt;/a&gt;. I&apos;ve attached a repair attempt of the files you provided (Dated Feb 26 2019 04:12:06 PM GMT-0800). Please replace them in your &lt;tt&gt;$dbpath&lt;/tt&gt; and let us know if it resolves the issue.&lt;/p&gt;

&lt;p&gt;Sincerely,&lt;br/&gt;
Eric&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="209935" name="WiredTiger" size="49" author="matthew.williams" created="Mon, 25 Feb 2019 19:43:53 +0000"/>
                            <attachment id="210049" name="WiredTiger.turtle" size="1002" author="eric.sedor@mongodb.com" created="Wed, 27 Feb 2019 00:12:05 +0000"/>
                            <attachment id="209934" name="WiredTiger.turtle" size="1014" author="matthew.williams" created="Mon, 25 Feb 2019 19:43:53 +0000"/>
                            <attachment id="210050" name="WiredTiger.wt" size="790528" author="eric.sedor@mongodb.com" created="Wed, 27 Feb 2019 00:12:06 +0000"/>
                            <attachment id="209933" name="WiredTiger.wt" size="790528" author="matthew.williams" created="Mon, 25 Feb 2019 19:43:53 +0000"/>
                            <attachment id="209932" name="WiredTigerLAS.wt" size="4096" author="matthew.williams" created="Mon, 25 Feb 2019 19:43:53 +0000"/>
                            <attachment id="209937" name="_mdb_catalog.wt" size="53248" author="matthew.williams" created="Mon, 25 Feb 2019 19:43:53 +0000"/>
                            <attachment id="209936" name="sizeStorer.wt" size="61440" author="matthew.williams" created="Mon, 25 Feb 2019 19:43:53 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>11.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 27 Feb 2019 00:13:00 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        4 years, 47 weeks, 6 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>eric.sedor@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            4 years, 47 weeks, 6 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>eric.sedor@mongodb.com</customfieldvalue>
            <customfieldvalue>matthew.williams</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|huonav:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|huee07:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10750" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Steps To Reproduce</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>&lt;p&gt;Run Mongodb on a VM&lt;/p&gt;

&lt;p&gt;Restart VM incorrectly to cause disk corruption&lt;/p&gt;

&lt;p&gt;Resolve Disk corruption&lt;/p&gt;

&lt;p&gt;Attempt to restart mongodb&lt;/p&gt;</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[eric.sedor@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|huo9k7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>