<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:23:40 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-10634] Failover doesn&apos;t occur on disk full and other non-crash errors</title>
                <link>https://jira.mongodb.org/browse/SERVER-10634</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Summary:&lt;/p&gt;

&lt;p&gt;Given a replica set with 3 or more nodes, if the PRIMARY node is shutdown, crashes, or becomes available due to network issues, the other nodes will proceed to elect a new PRIMARY and automatic failover occurs within seconds.&lt;/p&gt;

&lt;p&gt;However, in other error situations where the mongod process remains alive and continues to respond to heartbeats, failover will not happen, but write operations to the PRIMARY will fail, rendering the cluster unusable and de facto unavailable (for writes).&lt;/p&gt;

&lt;p&gt;An example of such error situation is a disk error such as disk full.&lt;/p&gt;
</description>
                <environment>Amazon Linux, official MongoDB.org packages</environment>
        <key id="87666">SERVER-10634</key>
            <summary>Failover doesn&apos;t occur on disk full and other non-crash errors</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="3">Duplicate</resolution>
                                        <assignee username="-1">Unassigned</assignee>
                                    <reporter username="henrik.ingo@mongodb.com">Henrik Ingo</reporter>
                        <labels>
                    </labels>
                <created>Wed, 28 Aug 2013 11:13:34 +0000</created>
                <updated>Wed, 10 Dec 2014 23:05:57 +0000</updated>
                            <resolved>Thu, 29 Aug 2013 04:38:07 +0000</resolved>
                                    <version>2.4.6</version>
                                                    <component>Replication</component>
                                        <votes>0</votes>
                                    <watches>2</watches>
                                                                                                                <comments>
                            <comment id="413750" author="dan@10gen.com" created="Thu, 29 Aug 2013 04:38:07 +0000"  >&lt;p&gt;duplicate of &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-9552&quot; title=&quot;when replica set member has full disk, step down to (sec|rec)?&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-9552&quot;&gt;SERVER-9552&lt;/a&gt;&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                            <outwardlinks description="duplicates">
                                        <issuelink>
            <issuekey id="74098">SERVER-9552</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Thu, 29 Aug 2013 04:38:07 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        10 years, 24 weeks, 6 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            10 years, 24 weeks, 6 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10000" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Old_Backport</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10000"><![CDATA[No]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>dan@mongodb.com</customfieldvalue>
            <customfieldvalue>henrik.ingo@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrmibj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrsjgf:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>80387</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10750" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Steps To Reproduce</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>&lt;p&gt;Setup a 3 node replica set. Use separate servers or at least different data disk for each node.&lt;/p&gt;

&lt;p&gt;Using the following script, insert data into a collection:&lt;/p&gt;

&lt;blockquote&gt;
&lt;p&gt;#!/bin/bash&lt;/p&gt;


&lt;p&gt;while true&lt;br/&gt;
do&lt;br/&gt;
  TIME=`date`&lt;br/&gt;
  echo &quot;db.test.insert( &lt;/p&gt;
&lt;div class=&quot;error&quot;&gt;&lt;span class=&quot;error&quot;&gt;Unknown macro: { time }&lt;/span&gt; &lt;/div&gt;
&lt;p&gt; )&quot; | mongo test&lt;br/&gt;
done&lt;/p&gt;&lt;/blockquote&gt;


&lt;p&gt;Use a tool like &lt;em&gt;dd if=/dev/zero of=/tmp/eatspace bs=1024 count=1024&lt;/em&gt; to fill the disk. Note that even after the disk is full, MongoDB will continue to successfully insert more data until the last 2 GB data file becomes full.&lt;/p&gt;


&lt;p&gt;&lt;b&gt;What actually happens:&lt;/b&gt;&lt;/p&gt;

&lt;p&gt;Observe following errors from the insert:&lt;/p&gt;

&lt;blockquote&gt;
&lt;p&gt;MongoDB shell version: 2.4.6&lt;br/&gt;
connecting to: test&lt;br/&gt;
Can&apos;t take a write lock while out of disk space&lt;br/&gt;
bye&lt;/p&gt;&lt;/blockquote&gt;


&lt;p&gt;And in the log:&lt;/p&gt;

&lt;blockquote&gt;
&lt;p&gt;Wed Aug 28 11:08:07.306 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; allocating new datafile /var/lib/mongo/test.2, filling with zeroes...&lt;br/&gt;
Wed Aug 28 11:08:07.307 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; FileAllocator: posix_fallocate failed: errno:28 No space left on device falling back&lt;br/&gt;
Wed Aug 28 11:08:07.307 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; error: failed to allocate new file: /var/lib/mongo/test.2 size: 268435456 failure creating new datafile; lseek failed for fd 22 with errno: errno:2 No such file or directory.  will try again in 10 seconds&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;&lt;b&gt;What should happen&lt;/b&gt;&lt;/p&gt;

&lt;p&gt;When failing to allocate a new datafile, the primary should step down and allow another node to become primary. In addition, it should go into a state where it cannot become primary again (for example, if it has a high priority) until the problem has been fixed.&lt;/p&gt;

&lt;p&gt;&lt;b&gt;Workarounds&lt;/b&gt;&lt;/p&gt;

&lt;p&gt;When noticing the failure, the DBA must call &lt;em&gt;rs.stepDown()&lt;/em&gt; or shut down the failing mongod process. &lt;em&gt;rs.stepDown()&lt;/em&gt; could also be called automatically from an application that receives disk full or other similar error message. In addition, it might make sense to set the node into hidden or priority=0 state until problem is fixed.&lt;/p&gt;</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hspc67:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>