<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:35:10 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-33971] Nodes in MongoDB sharded cluster crashes with Invariant failure oplogEntry.getWallClockTime()</title>
                <link>https://jira.mongodb.org/browse/SERVER-33971</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;We got strange error on our MongoDB cluster. It consists of 4 replicas with 3 nodes each.&lt;br/&gt;
Today, during half an hour nodes from 3 replicas began to crash in chain. Primary crashes -&amp;gt; one of secondaries becames primary -&amp;gt; in 2 minutes it crashes too -&amp;gt; new node goes primary -&amp;gt; again crashes in some minutes and so on. Nodes have autorestart, and in 10-15 minutes of chaining crashes all is ok and works normal. To mention, balancer was disabled at that time.&lt;br/&gt;
I&apos;ve attached two parts of logs with assertion from 2 nodes of one replica. If you need more logs or info - just ask.&lt;/p&gt;

&lt;p&gt;We use ubuntu 16.04 on servers and docker containers for mongo.&lt;br/&gt;
MongoDB version is 3.6.3&lt;/p&gt;</description>
                <environment></environment>
        <key id="513333">SERVER-33971</key>
            <summary>Nodes in MongoDB sharded cluster crashes with Invariant failure oplogEntry.getWallClockTime()</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="jack.mulrow@mongodb.com">Jack Mulrow</assignee>
                                    <reporter username="Ubus">Yuriy [X]</reporter>
                        <labels>
                            <label>SWNA</label>
                    </labels>
                <created>Mon, 19 Mar 2018 12:56:05 +0000</created>
                <updated>Sun, 29 Oct 2023 22:33:37 +0000</updated>
                            <resolved>Thu, 22 Mar 2018 00:30:41 +0000</resolved>
                                    <version>3.6.3</version>
                                    <fixVersion>3.6.4</fixVersion>
                    <fixVersion>3.7.4</fixVersion>
                                    <component>Sharding</component>
                                        <votes>0</votes>
                                    <watches>10</watches>
                                                                                                                <comments>
                            <comment id="1841487" author="xgen-internal-githook" created="Thu, 22 Mar 2018 00:27:32 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;email&apos;: &apos;jack.mulrow@mongodb.com&apos;, &apos;name&apos;: &apos;Jack Mulrow&apos;, &apos;username&apos;: &apos;jsmulrow&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-33971&quot; title=&quot;Nodes in MongoDB sharded cluster crashes with Invariant failure oplogEntry.getWallClockTime()&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-33971&quot;&gt;&lt;del&gt;SERVER-33971&lt;/del&gt;&lt;/a&gt; Generate session sentinel oplog entry with wall clock time&lt;/p&gt;

&lt;p&gt;(cherry picked from commit daa7dbf7e4564fc38b946416e3240caeb3c59b3a)&lt;br/&gt;
Branch: v3.6&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/f0575e255effe2c2e9d909187c8ee73ab2431b29&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/f0575e255effe2c2e9d909187c8ee73ab2431b29&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1841411" author="xgen-internal-githook" created="Wed, 21 Mar 2018 22:22:30 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;email&apos;: &apos;jack.mulrow@mongodb.com&apos;, &apos;name&apos;: &apos;Jack Mulrow&apos;, &apos;username&apos;: &apos;jsmulrow&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-33971&quot; title=&quot;Nodes in MongoDB sharded cluster crashes with Invariant failure oplogEntry.getWallClockTime()&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-33971&quot;&gt;&lt;del&gt;SERVER-33971&lt;/del&gt;&lt;/a&gt; Generate session sentinel oplog entry with wall clock time&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/daa7dbf7e4564fc38b946416e3240caeb3c59b3a&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/daa7dbf7e4564fc38b946416e3240caeb3c59b3a&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1839452" author="kaloian.manassiev" created="Tue, 20 Mar 2018 13:45:31 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=jack.mulrow&quot; class=&quot;user-hover&quot; rel=&quot;jack.mulrow&quot;&gt;jack.mulrow&lt;/a&gt;, the only place where it is possible to generate oplog entries &lt;a href=&quot;https://github.com/mongodb/mongo/blob/be85d6ca6905ec8d5215c5b25a41d73dd47696f6/src/mongo/db/s/session_catalog_migration_source.cpp#L93&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;without wallclock time&lt;/a&gt; is when we write the &lt;a href=&quot;https://github.com/mongodb/mongo/blob/be85d6ca6905ec8d5215c5b25a41d73dd47696f6/src/mongo/db/s/session_catalog_migration_source.cpp#L340&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;sentinel&lt;/a&gt; entry when a session has dropped off the end of the log. I believe this is what is happening in this situation - we are transferring entries for such a session and we are &lt;a href=&quot;https://github.com/mongodb/mongo/blob/be85d6ca6905ec8d5215c5b25a41d73dd47696f6/src/mongo/db/s/session_catalog_migration_destination.cpp#L246&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;invariant-ing&lt;/a&gt; too early.&lt;/p&gt;</comment>
                            <comment id="1838966" author="kaloian.manassiev" created="Mon, 19 Mar 2018 21:24:25 +0000"  >&lt;p&gt;Thank you for confirming, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=Ubus&quot; class=&quot;user-hover&quot; rel=&quot;Ubus&quot;&gt;Ubus&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;We have a theory about how this can happen that we are working on validating it. We will update the ticket once we have something more specific.&lt;/p&gt;

&lt;p&gt;Thank you again for the report!&lt;/p&gt;</comment>
                            <comment id="1838921" author="ubus" created="Mon, 19 Mar 2018 20:57:04 +0000"  >&lt;p&gt;Nope. We haven&apos;t used 3.6.0 rc4 or rc5 at all.&lt;br/&gt;
Last migration was from 3.6.2 to 3.6.3 in the end of February, 3 weeks ago. And from this time oplog should be rewritten many times, longest oplog length is 40+ hours. Size of oplog is 100Gb.&lt;br/&gt;
Sometimes DB is under heavy load, mostly because of bad disk IO.&lt;/p&gt;</comment>
                            <comment id="1838647" author="kaloian.manassiev" created="Mon, 19 Mar 2018 18:26:57 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=Ubus&quot; class=&quot;user-hover&quot; rel=&quot;Ubus&quot;&gt;Ubus&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Thank you very much for your report.&lt;/p&gt;

&lt;p&gt;The crash that you experienced indicates that chunk migration encountered an oplog entry, which contains retryable writes information, but no wallclock time component. Starting in version 3.6.0 we unconditionally &lt;a href=&quot;https://github.com/mongodb/mongo/blob/r3.6.0/src/mongo/db/repl/oplog.cpp#L352&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;write the wallclock time&lt;/a&gt; to all oplog entries, so this situation should theoretically not be possible.&lt;/p&gt;

&lt;p&gt;However I noticed that between 3.6.0 RC4 and RC5 we fixed &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-32027&quot; title=&quot;Ensure oplog entry has wallClockTime object regardless of FCV&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-32027&quot;&gt;&lt;del&gt;SERVER-32027&lt;/del&gt;&lt;/a&gt;, which was causing the wallclock entries to not always be written. Is it possible that at some point you had RC4 on your system and then eventually upgraded it to 3.6.3? This would have left inconsistent oplog entries and could cause the invariant in question.&lt;/p&gt;

&lt;p&gt;Best regards,&lt;br/&gt;
-Kal.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="181976" name="rs0-0.1.log" size="10717" author="Ubus" created="Mon, 19 Mar 2018 12:50:18 +0000"/>
                            <attachment id="181975" name="rs0-1.1.log" size="14887" author="Ubus" created="Mon, 19 Mar 2018 12:50:22 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>6.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="15141"><![CDATA[v3.6]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 19 Mar 2018 13:53:23 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        5 years, 47 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            5 years, 47 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>jack.mulrow@mongodb.com</customfieldvalue>
            <customfieldvalue>kaloian.manassiev@mongodb.com</customfieldvalue>
            <customfieldvalue>Ubus</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htt01r:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|htkbqn:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htsm93:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>