<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:16:55 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-8242] assertion failures (btree) during repair</title>
                <link>https://jira.mongodb.org/browse/SERVER-8242</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Was running a repair with the following command and got this assertion error in the log of 1 out of 3 shards. The other shards had a different assertion error, which I filed a different bug for:&lt;/p&gt;

&lt;p&gt;ubuntu@mongo3:~$ mongod --version&lt;br/&gt;
db version v2.2.0, pdfile version 4.5&lt;br/&gt;
Fri Jan 18 20:56:42 git version: f5e83eae9cfbec7fb7a071321928f00d1b0c5207&lt;/p&gt;

&lt;p&gt;sudo -u mongodb mongod --dbpath=/db/mongodb --repair --repairpath=/export/mongodb --nojournal&lt;/p&gt;

&lt;p&gt;Fri Jan 18 03:25:59 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             47127700/861940368      5%&lt;br/&gt;
Fri Jan 18 03:26:09 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             47232800/861940368      5%&lt;br/&gt;
Fri Jan 18 03:26:19 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             47342500/861940368      5%&lt;br/&gt;
Fri Jan 18 03:26:29 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             47451000/861940368      5%&lt;br/&gt;
Fri Jan 18 03:26:39 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             47560200/861940368      5%&lt;br/&gt;
Fri Jan 18 03:26:49 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             47665300/861940368      5%&lt;br/&gt;
Fri Jan 18 03:26:59 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             47774400/861940368      5%&lt;br/&gt;
Fri Jan 18 03:27:09 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             47881800/861940368      5%&lt;br/&gt;
Fri Jan 18 03:27:19 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             47988500/861940368      5%&lt;br/&gt;
Fri Jan 18 03:27:29 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             48097900/861940368      5%&lt;br/&gt;
Fri Jan 18 03:27:39 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             48207400/861940368      5%&lt;br/&gt;
Fri Jan 18 03:27:49 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             48313200/861940368      5%&lt;br/&gt;
Fri Jan 18 03:27:59 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             48422400/861940368      5%&lt;br/&gt;
Fri Jan 18 03:28:00 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;  pb3 Assertion failure la &amp;lt;= 0xffffff src/mongo/db/btree.h 243&lt;br/&gt;
0xade6e1 0x803dfd 0x683ebb 0xb1e868 0xae7d19 0xae15ed 0xae46e7 0x747ed3 0x7490bd 0xae2c0d 0x737f9c 0x8535bc 0x8560c9 0x856269 0x74037a 0x5596d8 0x55b78d 0x55c6f0 0x55d2dd 0x563dbe&lt;br/&gt;
 mongod(_ZN5mongo15printStackTraceERSo+0x21) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xade6e1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo12verifyFailedEPKcS1_j+0xfd) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x803dfd&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(&lt;em&gt;ZN5mongo12BucketBasicsINS_12BtreeData_V1EE9_pushBackENS_7DiskLocERKNS_5KeyV1ERKNS_8OrderingES3&lt;/em&gt;+0x8ab) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x683ebb&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo12BtreeBuilderINS_12BtreeData_V1EE6addKeyERNS_7BSONObjENS_7DiskLocE+0x2a8) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xb1e868&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo24buildBottomUpPhases2And3INS_12BtreeData_V1EEEvbRNS_12IndexDetailsERNS_21BSONObjExternalSorterEbRSt3setINS_7DiskLocESt4lessIS7_ESaIS7_EEPNS_5CurOpEPNS_12SortPhaseOneERNS_19ProgressMeterHolderERNS_5TimerE+0x1f9) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xae7d19&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo14fastBuildIndexEPKcPNS_16NamespaceDetailsERNS_12IndexDetailsEi+0xc9d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xae15ed&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo12buildAnIndexESsPNS_16NamespaceDetailsERNS_12IndexDetailsEib+0x217) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xae46e7&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo16insert_makeIndexEPNS_16NamespaceDetailsERKSsRKNS_7DiskLocE+0x1f3) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x747ed3&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo11DataFileMgr6insertEPKcPKvibbPb+0x93d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7490bd&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo17ensureHaveIdIndexEPKc+0x7ad) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xae2c0d&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo21ensureIdIndexForNewNsEPKc+0x9c) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x737f9c&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo6Cloner2goEPKcRKNS_12CloneOptionsERSt3setISsSt4lessISsESaISsEERSsPi+0xd1c) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x8535bc&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo6Cloner2goEPKcRSsRKSsbbbbbbPi+0x139) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x8560c9&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo9cloneFromEPKcRSsRKSsbbbbbbPi+0x59) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x856269&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo14repairDatabaseESsRSsbb+0x52a) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x74037a&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo11doDBUpgradeERKSsSsPNS_14DataFileHeaderE+0x68) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x5596d8&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod() &lt;span class=&quot;error&quot;&gt;&amp;#91;0x55b78d&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo14_initAndListenEi+0x480) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x55c6f0&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo13initAndListenEi+0x1d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x55d2dd&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod() &lt;span class=&quot;error&quot;&gt;&amp;#91;0x563dbe&amp;#93;&lt;/span&gt;&lt;br/&gt;
Fri Jan 18 03:28:00 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; Assertion: 16394:reading doc for external sort failed:errno:2 No such file or directory&lt;br/&gt;
0xade6e1 0x8036eb 0x80386c 0x909535 0x909c10 0xae7bff 0xae15ed 0xae46e7 0x747ed3 0x7490bd 0xae2c0d 0x737f9c 0x8535bc 0x8560c9 0x856269 0x74037a 0x5596d8 0x55b78d 0x55c6f0 0x55d2dd&lt;br/&gt;
 mongod(_ZN5mongo15printStackTraceERSo+0x21) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xade6e1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo11msgassertedEiPKc+0x9b) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x8036eb&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod() &lt;span class=&quot;error&quot;&gt;&amp;#91;0x80386c&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo21BSONObjExternalSorter12FileIterator4nextEv+0xc5) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x909535&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo21BSONObjExternalSorter8Iterator4nextEv+0x220) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x909c10&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo24buildBottomUpPhases2And3INS_12BtreeData_V1EEEvbRNS_12IndexDetailsERNS_21BSONObjExternalSorterEbRSt3setINS_7DiskLocESt4lessIS7_ESaIS7_EEPNS_5CurOpEPNS_12SortPhaseOneERNS_19ProgressMeterHolderERNS_5TimerE+0xdf) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xae7bff&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo14fastBuildIndexEPKcPNS_16NamespaceDetailsERNS_12IndexDetailsEi+0xc9d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xae15ed&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo12buildAnIndexESsPNS_16NamespaceDetailsERNS_12IndexDetailsEib+0x217) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xae46e7&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo16insert_makeIndexEPNS_16NamespaceDetailsERKSsRKNS_7DiskLocE+0x1f3) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x747ed3&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo11DataFileMgr6insertEPKcPKvibbPb+0x93d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7490bd&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo17ensureHaveIdIndexEPKc+0x7ad) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xae2c0d&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo21ensureIdIndexForNewNsEPKc+0x9c) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x737f9c&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo6Cloner2goEPKcRKNS_12CloneOptionsERSt3setISsSt4lessISsESaISsEERSsPi+0xd1c) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x8535bc&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo6Cloner2goEPKcRSsRKSsbbbbbbPi+0x139) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x8560c9&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo9cloneFromEPKcRSsRKSsbbbbbbPi+0x59) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x856269&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo14repairDatabaseESsRSsbb+0x52a) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x74037a&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo11doDBUpgradeERKSsSsPNS_14DataFileHeaderE+0x68) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x5596d8&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod() &lt;span class=&quot;error&quot;&gt;&amp;#91;0x55b78d&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo14_initAndListenEi+0x480) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x55c6f0&amp;#93;&lt;/span&gt;&lt;br/&gt;
 mongod(_ZN5mongo13initAndListenEi+0x1d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x55d2dd&amp;#93;&lt;/span&gt;&lt;br/&gt;
Fri Jan 18 03:28:08 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; exception in initAndListen: 16394 reading doc for external sort failed:errno:2 No such file or directory, terminating&lt;br/&gt;
Fri Jan 18 03:28:08 dbexit:&lt;br/&gt;
Fri Jan 18 03:28:08 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; shutdown: going to close listening sockets...&lt;br/&gt;
Fri Jan 18 03:28:08 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; shutdown: going to flush diaglog...&lt;br/&gt;
Fri Jan 18 03:28:08 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; shutdown: going to close sockets...&lt;br/&gt;
Fri Jan 18 03:28:08 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; shutdown: waiting for fs preallocator...&lt;br/&gt;
Fri Jan 18 03:28:08 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; shutdown: closing all files...&lt;br/&gt;
Fri Jan 18 03:28:10 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             119/824 14%&lt;br/&gt;
Fri Jan 18 03:28:12 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             402/824 48%&lt;br/&gt;
Fri Jan 18 03:28:14 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             448/824 54%&lt;br/&gt;
Fri Jan 18 03:28:16 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             455/824 55%&lt;br/&gt;
Fri Jan 18 03:28:18 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             464/824 56%&lt;br/&gt;
Fri Jan 18 03:28:20 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             573/824 69%&lt;br/&gt;
Fri Jan 18 03:28:22 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             580/824 70%&lt;br/&gt;
Fri Jan 18 03:28:24 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             605/824 73%&lt;br/&gt;
Fri Jan 18 03:28:26 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt;             671/824 81%&lt;br/&gt;
Fri Jan 18 03:28:27 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; closeAllFiles() finished&lt;br/&gt;
Fri Jan 18 03:28:27 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; shutdown: removing fs lock...&lt;br/&gt;
Fri Jan 18 03:28:27 dbexit: really exiting now&lt;/p&gt;</description>
                <environment>ubuntu on EC2</environment>
        <key id="62606">SERVER-8242</key>
            <summary>assertion failures (btree) during repair</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="4">Incomplete</resolution>
                                        <assignee username="-1">Unassigned</assignee>
                                    <reporter username="zacwitte">Zac Witte</reporter>
                        <labels>
                    </labels>
                <created>Fri, 18 Jan 2013 21:01:33 +0000</created>
                <updated>Wed, 10 Dec 2014 23:18:08 +0000</updated>
                            <resolved>Mon, 3 Jun 2013 00:05:57 +0000</resolved>
                                    <version>2.2.0</version>
                                                    <component>Admin</component>
                                        <votes>0</votes>
                                    <watches>4</watches>
                                                                                                                <comments>
                            <comment id="351236" author="dan@10gen.com" created="Mon, 3 Jun 2013 00:05:57 +0000"  >&lt;p&gt;Looks to be the same problem as &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-8241&quot; title=&quot;assertion failures (saveerrmsg) during repair&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-8241&quot;&gt;&lt;del&gt;SERVER-8241&lt;/del&gt;&lt;/a&gt; in which the index build in the &lt;tt&gt;dbpath&lt;/tt&gt; caused disk space issues.  Can&apos;t say for sure without more data.&lt;/p&gt;</comment>
                            <comment id="294480" author="aaron" created="Thu, 21 Mar 2013 00:34:31 +0000"  >&lt;p&gt;Hi Zac,&lt;/p&gt;

&lt;p&gt;One other thing to be aware of is that the temporary external sort files go inside _tmp within your dbpath not your repair path.  So a disk error including out of disk space on the partition containing /db/mongodb could cause this issue.&lt;/p&gt;

&lt;p&gt;We have&lt;br/&gt;
&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-8917&quot; title=&quot;buildBottomUpPhases2And3 should check assertion code when detecting a dup&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-8917&quot;&gt;&lt;del&gt;SERVER-8917&lt;/del&gt;&lt;/a&gt;&lt;br/&gt;
&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-8449&quot; title=&quot;Index build errors during mongod --repair swallowed, not reported.&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-8449&quot;&gt;&lt;del&gt;SERVER-8449&lt;/del&gt;&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;to improve reporting when there is an error during a repair operation.&lt;/p&gt;

&lt;p&gt;For this ticket and &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-8241&quot; title=&quot;assertion failures (saveerrmsg) during repair&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-8241&quot;&gt;&lt;del&gt;SERVER-8241&lt;/del&gt;&lt;/a&gt;, if you know your index sizes and amount of free space on /db/mongodb at the time of the error we may be able to determine whether a repair run under these conditions could have exhausted disk space.&lt;/p&gt;</comment>
                            <comment id="248190" author="zacwitte" created="Thu, 24 Jan 2013 01:25:17 +0000"  >&lt;p&gt;OK so it sounds like mongod needs to be more robust in how it reads in files and retries on failure. At the very least it needs more user-friendly error messages.&lt;/p&gt;</comment>
                            <comment id="248061" author="aaron" created="Wed, 23 Jan 2013 22:12:22 +0000"  >&lt;p&gt;Hi Zac,&lt;/p&gt;

&lt;p&gt;Here&apos;s some analysis of the two assertions described in the log.  But first some background info:&lt;/p&gt;

&lt;p&gt;A foreground index build is implemented in three phases:&lt;/p&gt;

&lt;p&gt;1) The whole collection is scanned, and for each document in the collection the set of keys that will appear in the index are extracted from the document.  The keys for all documents, along with the locations of these documents, are saved into a set of temporary external sort files in a temporary directory ( named _tmp ) within your dbpath.  (The keys within each external sort file are sorted, but there is no sorting relationship between the different files.)&lt;/p&gt;

&lt;p&gt;2) A merge sort over the external sort files is used to iterate over all the keys in the external sort files, in order.  The keys are used to build the leaves of a new btree.&lt;/p&gt;

&lt;p&gt;3) The internal nodes of the btree are built up based on the leaves of the btree.&lt;/p&gt;


&lt;p&gt;The first error in the log, about &quot;la &amp;lt;= 0xffffff&quot; is occurring because an invalid document location is being passed from an external sort file to a new btree leaf as part of step 2.&lt;/p&gt;

&lt;p&gt;The second error in the log, about &quot;reading doc for external sort failed&quot; describes an error reading an external sort file.  It looks like it&apos;s occurring while reading an index key from the external sort file.  It looks like part of the key is read but there is an error while reading the rest of the key.&lt;/p&gt;

&lt;p&gt;I believe what&apos;s happening is that the implementation assumes the first error is a dup key error, even though it seems to result from a bad read of the external sort file.  Then it continues trying to read from the external sort file until it sees the second error and at that point fails.&lt;/p&gt;</comment>
                            <comment id="246876" author="zacwitte" created="Tue, 22 Jan 2013 20:15:58 +0000"  >&lt;p&gt;I no longer have the full logs, but I have the excerpts where the exceptions occurred. I&apos;ll attach them here. I think everything looked normal until this exception. The /export partition did not run out of disk space, but it is possible that the EBS volumes or the network connection had a hiccup. That kind of thing is always possible with EC2. It is strange that all 3 servers had these assertion errors at different times, though.&lt;/p&gt;

&lt;p&gt;I noticed this in my syslog but I can&apos;t be sure if it&apos;s related since there&apos;s no way to match timestamps:&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;11797218.035099&amp;#93;&lt;/span&gt; md/raid0:md100: md_size is 10737408000 sectors.&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11797218.035101&amp;#93;&lt;/span&gt; ******* md100 configuration *********&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11797218.035103&amp;#93;&lt;/span&gt; zone0=&lt;span class=&quot;error&quot;&gt;&amp;#91;xvdi1/xvdi2/xvdi3/xvdi4/xvdi5/&amp;#93;&lt;/span&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11797218.035109&amp;#93;&lt;/span&gt;         zone offset=0kb device offset=0kb size=5368704000kb&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11797218.035111&amp;#93;&lt;/span&gt; **********************************&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11797218.035112&amp;#93;&lt;/span&gt; &lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11797218.035126&amp;#93;&lt;/span&gt; md100: detected capacity change from 0 to 5497552896000&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11797218.040469&amp;#93;&lt;/span&gt;  md100: unknown partition table&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11797321.696348&amp;#93;&lt;/span&gt; XFS (md100): Mounting Filesystem&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11797321.871849&amp;#93;&lt;/span&gt; XFS (md100): Ending clean mount&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207563&amp;#93;&lt;/span&gt; INFO: task kworker/0:1:23258 blocked for more than 120 seconds.&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207579&amp;#93;&lt;/span&gt; &quot;echo 0 &amp;gt; /proc/sys/kernel/hung_task_timeout_secs&quot; disables this message.&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207589&amp;#93;&lt;/span&gt; kworker/0:1     D ffff881116ee2940     0 23258      2 0x00000000&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207596&amp;#93;&lt;/span&gt;  ffff88001c45bcc0 0000000000000246 ffff88001c45ba44 0000000000000000&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207603&amp;#93;&lt;/span&gt;  ffff88001c45bfd8 ffff88001c45bfd8 ffff88001c45bfd8 0000000000012940&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207610&amp;#93;&lt;/span&gt;  ffff8810a3b00000 ffff8810a0d2dc00 ffffffff8160380e ffff8810a1337c18&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207615&amp;#93;&lt;/span&gt; Call Trace:&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207627&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8160380e&amp;gt;&amp;#93;&lt;/span&gt; ? _raw_spin_unlock_irqrestore+0x1e/0x30&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207661&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa00df811&amp;gt;&amp;#93;&lt;/span&gt; _xfs_log_force_lsn+0x171/0x2e0 &lt;span class=&quot;error&quot;&gt;&amp;#91;xfs&amp;#93;&lt;/span&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207668&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff810570e0&amp;gt;&amp;#93;&lt;/span&gt; ? try_to_wake_up+0x200/0x200&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207685&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa00ececd&amp;gt;&amp;#93;&lt;/span&gt; _xfs_trans_commit+0x29d/0x2b0 &lt;span class=&quot;error&quot;&gt;&amp;#91;xfs&amp;#93;&lt;/span&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207702&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa0100c30&amp;gt;&amp;#93;&lt;/span&gt; ? xfs_sync_inode_attr+0xf0/0xf0 &lt;span class=&quot;error&quot;&gt;&amp;#91;xfs&amp;#93;&lt;/span&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207720&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa00d00c4&amp;gt;&amp;#93;&lt;/span&gt; xfs_fs_log_dummy+0x64/0x90 &lt;span class=&quot;error&quot;&gt;&amp;#91;xfs&amp;#93;&lt;/span&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207740&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffffa0100cb4&amp;gt;&amp;#93;&lt;/span&gt; xfs_sync_worker+0x84/0x90 &lt;span class=&quot;error&quot;&gt;&amp;#91;xfs&amp;#93;&lt;/span&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207749&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8107af3a&amp;gt;&amp;#93;&lt;/span&gt; process_one_work+0x11a/0x480&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207755&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8110af47&amp;gt;&amp;#93;&lt;/span&gt; ? mempool_free_slab+0x17/0x20&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207758&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8107bcdf&amp;gt;&amp;#93;&lt;/span&gt; worker_thread+0x15f/0x360&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207761&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8107bb80&amp;gt;&amp;#93;&lt;/span&gt; ? manage_workers.isra.30+0x130/0x130&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207766&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff810807bc&amp;gt;&amp;#93;&lt;/span&gt; kthread+0x8c/0xa0&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207770&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8160cae4&amp;gt;&amp;#93;&lt;/span&gt; kernel_thread_helper+0x4/0x10&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207774&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8160bbe3&amp;gt;&amp;#93;&lt;/span&gt; ? int_ret_from_sys_call+0x7/0x1b&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207778&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff81603b61&amp;gt;&amp;#93;&lt;/span&gt; ? retint_restore_args+0x5/0x6&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;11945966.207781&amp;#93;&lt;/span&gt;  &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;lt;ffffffff8160cae0&amp;gt;&amp;#93;&lt;/span&gt; ? gs_change+0x13/0x13&lt;/p&gt;</comment>
                            <comment id="246818" author="aaron" created="Tue, 22 Jan 2013 19:22:50 +0000"  >&lt;p&gt;Hi Zac - Can you send the full log?  Also, is it possible there was a disk error or you ran out of disk space while the repair was running?&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="62605">SERVER-8241</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="67821">SERVER-8917</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="23382" name="mongo1_repair.log" size="3952" author="zacwitte" created="Tue, 22 Jan 2013 20:16:23 +0000"/>
                            <attachment id="23383" name="mongo2_repair.log" size="4181" author="zacwitte" created="Tue, 22 Jan 2013 20:16:23 +0000"/>
                            <attachment id="23384" name="mongo3_repair.log" size="5899" author="zacwitte" created="Tue, 22 Jan 2013 20:16:23 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>6.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 22 Jan 2013 19:22:50 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        10 years, 37 weeks, 3 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            10 years, 37 weeks, 3 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10000" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Old_Backport</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10000"><![CDATA[No]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>aaron</customfieldvalue>
            <customfieldvalue>dan@mongodb.com</customfieldvalue>
            <customfieldvalue>zacwitte</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrnasf:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrlp9z:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>40148</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hse4sn:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>