<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:31:16 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-53566] Investigate and reproduce &quot;opCtx != nullptr &amp;&amp; _opCtx == nullptr&quot; invariant</title>
                <link>https://jira.mongodb.org/browse/SERVER-53566</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;There seems to be an unknown issue causing a particular invariant to occasionally get hit, which results in nodes unexpectedly terminating. The invariant is improved as part of &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-49468&quot; title=&quot;Invalidate previous OperationContext when a new OperationContext is created&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-49468&quot;&gt;&lt;del&gt;SERVER-49468&lt;/del&gt;&lt;/a&gt; but this ticket is to track a root cause investigation.&lt;/p&gt;

&lt;p&gt;Someone needs to spend some dedicated time in order to try and unwind what may be happening here and determine the steps required to reproduce this particular bug.&#160;&lt;/p&gt;

&lt;p&gt;Acceptance Criteria:&#160; Either:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;A set of procedures that a user might be able to implement which causes the invariants to trigger.&#160;&lt;/li&gt;
	&lt;li&gt;A plan for how to add additional, backportable, diagnostic data that will allow us to properly track down the cause of this invariant being triggered.&#160;&lt;/li&gt;
&lt;/ul&gt;
</description>
                <environment></environment>
        <key id="1580342">SERVER-53566</key>
            <summary>Investigate and reproduce &quot;opCtx != nullptr &amp;&amp; _opCtx == nullptr&quot; invariant</summary>
                <type id="3" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14718&amp;avatarType=issuetype">Task</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="tyler.seip@mongodb.com">Tyler Seip</assignee>
                                    <reporter username="matthew.tretin@mongodb.com">Matthew Tretin</reporter>
                        <labels>
                            <label>sa-groomed</label>
                    </labels>
                <created>Mon, 4 Jan 2021 20:48:02 +0000</created>
                <updated>Sun, 29 Oct 2023 21:59:10 +0000</updated>
                            <resolved>Tue, 23 Mar 2021 16:22:57 +0000</resolved>
                                                    <fixVersion>4.9.0</fixVersion>
                    <fixVersion>4.4.5</fixVersion>
                    <fixVersion>4.0.24</fixVersion>
                    <fixVersion>4.2.14</fixVersion>
                                                        <votes>8</votes>
                                    <watches>45</watches>
                                                                                                                <comments>
                            <comment id="3681931" author="xgen-internal-githook" created="Wed, 24 Mar 2021 17:03:58 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Tyler Seip&apos;, &apos;email&apos;: &apos;Tyler.Seip@mongodb.com&apos;, &apos;username&apos;: &apos;tseip-mongo&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-53566&quot; title=&quot;Investigate and reproduce &amp;quot;opCtx != nullptr &amp;amp;&amp;amp; _opCtx == nullptr&amp;quot; invariant&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-53566&quot;&gt;&lt;del&gt;SERVER-53566&lt;/del&gt;&lt;/a&gt;: Protect ServiceContext from issuing duplicate operation IDs&lt;br/&gt;
Branch: v4.0&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/fbf13d24435cc05a04cbc56e560397e8b52d4ac9&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/fbf13d24435cc05a04cbc56e560397e8b52d4ac9&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3680609" author="xgen-internal-githook" created="Wed, 24 Mar 2021 00:40:20 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Tyler Seip&apos;, &apos;email&apos;: &apos;Tyler.Seip@mongodb.com&apos;, &apos;username&apos;: &apos;tseip-mongo&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-53566&quot; title=&quot;Investigate and reproduce &amp;quot;opCtx != nullptr &amp;amp;&amp;amp; _opCtx == nullptr&amp;quot; invariant&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-53566&quot;&gt;&lt;del&gt;SERVER-53566&lt;/del&gt;&lt;/a&gt;: Protect ServiceContext from issuing duplicate operation IDs&lt;br/&gt;
Branch: v4.2&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/2d30db50b66fb8c31432b6f622e010a5b7bce256&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/2d30db50b66fb8c31432b6f622e010a5b7bce256&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3680487" author="xgen-internal-githook" created="Tue, 23 Mar 2021 22:34:35 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Tyler Seip&apos;, &apos;email&apos;: &apos;Tyler.Seip@mongodb.com&apos;, &apos;username&apos;: &apos;tseip-mongo&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-53566&quot; title=&quot;Investigate and reproduce &amp;quot;opCtx != nullptr &amp;amp;&amp;amp; _opCtx == nullptr&amp;quot; invariant&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-53566&quot;&gt;&lt;del&gt;SERVER-53566&lt;/del&gt;&lt;/a&gt;: Protect ServiceContext from issuing duplicate operation IDs&lt;br/&gt;
Branch: v4.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/5fe45689b6eb936190203c66b506d5d042a76eb0&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/5fe45689b6eb936190203c66b506d5d042a76eb0&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3680028" author="dan@10gen.com" created="Tue, 23 Mar 2021 18:53:36 +0000"  >&lt;p&gt;Yes, BACKPORT tickets are private, as they&apos;re just used for internal bookkeeping. The fixVersion on the ticket will be updated and the commit message will be included as a comment in this ticket (with the branch name in the message) when the backport lands.&lt;/p&gt;</comment>
                            <comment id="3679598" author="JIRAUSER1257743" created="Tue, 23 Mar 2021 16:29:09 +0000"  >&lt;p&gt;Ah, it must be the case that they&apos;re only visible to employees (the 4.4 ticket for example is BACKPORT-8675) - the backports themselves are working their way through the commit queues now and should be committed in a day or two.&lt;/p&gt;</comment>
                            <comment id="3679585" author="sz" created="Tue, 23 Mar 2021 16:27:06 +0000"  >&lt;p&gt;@Tyler Seip, I don&apos;t see backporting tickets linked to the issue. Are they visible only to MongoDB employees?&lt;/p&gt;</comment>
                            <comment id="3679573" author="JIRAUSER1257743" created="Tue, 23 Mar 2021 16:22:57 +0000"  >&lt;p&gt;This has been fixed on master and is being backported on each of the respective linked backporting tickets. Closing this as resolved.&lt;/p&gt;</comment>
                            <comment id="3677416" author="xgen-internal-githook" created="Mon, 22 Mar 2021 19:20:17 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Tyler Seip&apos;, &apos;email&apos;: &apos;Tyler.Seip@mongodb.com&apos;, &apos;username&apos;: &apos;tseip-mongo&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-53566&quot; title=&quot;Investigate and reproduce &amp;quot;opCtx != nullptr &amp;amp;&amp;amp; _opCtx == nullptr&amp;quot; invariant&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-53566&quot;&gt;&lt;del&gt;SERVER-53566&lt;/del&gt;&lt;/a&gt;: Protect ServiceContext from issuing duplicate operation IDs&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/dbb6b0d496badf3e841ab41270e23cc0e4dc2ea4&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/dbb6b0d496badf3e841ab41270e23cc0e4dc2ea4&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="3670591" author="bozaro@gmail.com" created="Thu, 18 Mar 2021 06:43:43 +0000"  >&lt;p&gt;We &lt;a href=&quot;https://github.com/mongodb/mongo/pull/1400&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;made a PR&lt;/a&gt; against v4.4 branch with check and skip already used opId to avoid _nextOpId wraparound issue.&lt;/p&gt;</comment>
                            <comment id="3660202" author="bozaro@gmail.com" created="Thu, 11 Mar 2021 20:28:50 +0000"  >&lt;p&gt;I sorry, but look like Mongos 4.4.5 is not released yet.&lt;/p&gt;</comment>
                            <comment id="3660151" author="JIRAUSER1257743" created="Thu, 11 Mar 2021 20:10:41 +0000"  >&lt;p&gt;In response to this&#160;comment, we currently think that the server only needs enough uptime to trigger this issue, but that the large amount of operations emitted by events such as stepDown/network issues make it more likely that this issue will occur contemporaneously with them.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;In response to this&#160;comment, the solution from that ticket will prevent the server from crashing entirely but does not fix the underlying issue, which will be fixed and back ported on this ticket.&lt;/p&gt;</comment>
                            <comment id="3653980" author="JIRAUSER1258647" created="Tue, 9 Mar 2021 10:55:33 +0000"  >&lt;p&gt;Hi !&lt;/p&gt;

&lt;p&gt;As said in&#160;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-52735&quot; title=&quot;mongodb crash with &amp;quot;Invariant failure&amp;quot; opCtx != nullptr &amp;amp;&amp;amp; _opCtx == nullptr&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-52735&quot;&gt;&lt;del&gt;SERVER-52735&lt;/del&gt;&lt;/a&gt; ticket, we encountered the same issue (&#160;&quot;ctx&quot;:&quot;monitoring-keys-for-HMAC&quot;,&quot;msg&quot;:&quot;Invariant failure&quot;,&quot;attr&quot;:{&quot;expr&quot;:&quot;opCtx != nullptr &amp;amp;&amp;amp; _opCtx == nullptr&quot;,&quot;file&quot;:&quot;src/mongo/db/client.cpp&quot;,&quot;line&quot;:126}} ) on the replicates of one of our mongo bases, two days apart between replicate 1 (crash on March 8, at 23:56:15 UTC) and replicate 2 (crash this morning, March 9 at 00:04:25 UTC)&#160;with exactly the same log/stacktrace as the one present in the description of&#160;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-52735&quot; title=&quot;mongodb crash with &amp;quot;Invariant failure&amp;quot; opCtx != nullptr &amp;amp;&amp;amp; _opCtx == nullptr&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-52735&quot;&gt;&lt;del&gt;SERVER-52735&lt;/del&gt;&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;We&#160;saw that one of the related tickets (&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-49468&quot; title=&quot;Invalidate previous OperationContext when a new OperationContext is created&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-49468&quot;&gt;&lt;del&gt;SERVER-49468&lt;/del&gt;&lt;/a&gt;) had been marked as fixed in version 4.4.5, do you know if the update to this version will fix the problem or not?&#160;&lt;/p&gt;

&lt;p&gt;Thanks&lt;/p&gt;</comment>
                            <comment id="3652688" author="JIRAUSER1254842" created="Mon, 8 Mar 2021 18:41:16 +0000"  >&lt;p&gt;&amp;gt;&#160;Does the server need to have enough uptime AND a network issue to trigger the crash?&lt;/p&gt;

&lt;p&gt;There is also 20day forced refresh period in&#160;KeysCollectionManager::PeriodicRunner::_doPeriodicRefresh (&quot;ctx&quot;:&#160;&quot;monitoring-keys-for-HMAC&quot; in log) (as found and mentioned by Sergey Zagursky in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-49468&quot; title=&quot;Invalidate previous OperationContext when a new OperationContext is created&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-49468&quot;&gt;&lt;del&gt;SERVER-49468&lt;/del&gt;&lt;/a&gt;) .&#160; Looks like 20 days is often enough for wraparound to happen.&lt;/p&gt;</comment>
                            <comment id="3652647" author="bigbourin@gmail.com" created="Mon, 8 Mar 2021 18:25:29 +0000"  >&lt;p&gt;Hi, if this explanation is confirmed, as we noticed that the crash occurs when the server is stepping down or suffering from network issues, what would be the link between the two issues (long uptime with integer overflow and network issue)? Does the server need to have enough uptime AND a network issue to trigger the crash? If there&apos;s no network issues / stepDown, would the server leave with this integer overflow?&lt;br/&gt;
Thanks for the precisions!&lt;/p&gt;</comment>
                            <comment id="3652343" author="JIRAUSER1257743" created="Mon, 8 Mar 2021 16:53:02 +0000"  >&lt;p&gt;We currently suspect that &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-53566?focusedCommentId=3651171&amp;amp;page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-3651171&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;this comment&lt;/a&gt; has correctly identified the issue (thanks &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=funny.falcon%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;funny.falcon@gmail.com&quot;&gt;funny.falcon@gmail.com&lt;/a&gt;!). &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=amirsaman.memaripour&quot; class=&quot;user-hover&quot; rel=&quot;amirsaman.memaripour&quot;&gt;amirsaman.memaripour&lt;/a&gt; wrote a unit test which is able to repro this condition on wraparound. We&apos;re currently planning on leaving OperationId a uint32_t (as it is directly exposed in multiple parts of the system and so is not just implementation details) but adding some slow-path checking to _clientByOperationId in the event of a collision. This should be sufficient to resolve the issue within the week assuming performance holds with the additional checks.&lt;/p&gt;</comment>
                            <comment id="3652310" author="ben.caimano" created="Mon, 8 Mar 2021 16:43:53 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=rectalogic&quot; class=&quot;user-hover&quot; rel=&quot;rectalogic&quot;&gt;rectalogic&lt;/a&gt;, that is a good data point to have, thank you!&lt;/p&gt;</comment>
                            <comment id="3652296" author="rectalogic" created="Mon, 8 Mar 2021 16:39:47 +0000"  >&lt;p&gt;Both times we experienced this crash I believe our uptime was on the order of 6 months or more when we stepped down the primary and crashed.&lt;/p&gt;</comment>
                            <comment id="3652254" author="ben.caimano" created="Mon, 8 Mar 2021 16:30:39 +0000"  >&lt;p&gt;Back of the envelope calculation says &lt;tt&gt;(2^32-1)/1000(ms/s)/60(s/min)/60(min/h)/24(h/day) ~= 50days&lt;/tt&gt;, so we&apos;d need to generate a new operation every millisecond for 50 days. If we rephrase this to a scale like 3 months, that&apos;s not impossible. There are certainly going to be issues with bson type conversion for our RPC so we&apos;d probably want to put this under a flag to allow users with especially long uptimes to opt in. Checking for operation id integer overflow under the hood is probably significantly less intrusive as a first step.&lt;/p&gt;</comment>
                            <comment id="3651680" author="JIRAUSER1254842" created="Mon, 8 Mar 2021 13:11:02 +0000"  >&lt;p&gt;I&apos;ve &lt;a href=&quot;https://github.com/mongodb/mongo/pull/1399&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;made a PR&lt;/a&gt; against v4.4 branch with changing OperationId to uint64_t. I believe it should fix the issue.&lt;/p&gt;</comment>
                            <comment id="3651171" author="JIRAUSER1254842" created="Sun, 7 Mar 2021 19:12:28 +0000"  >&lt;p&gt;All failing traces are in long-living loops in non-client threads:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;KeysCollectionManager::PeriodicRunner::_doPeriodicRefresh&lt;/li&gt;
	&lt;li&gt;WaitForMajorityService::_periodicallyWaitForMajority&lt;/li&gt;
	&lt;li&gt;TopologyVersionObserver::_workerThreadBody&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;There is one of possible scenarios:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;when `UniqueOperationContext` is destroyed, it calls `OperationContextDeleter:operator()`&#160;&lt;a href=&quot;https://github.com/mongodb/mongo/blob/r4.4.3/src/mongo/db/service_context.cpp#L277&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/blob/r4.4.3/src/mongo/db/service_context.cpp#L277&lt;/a&gt;&lt;/li&gt;
	&lt;li&gt;it calls `ServiceContext::_delistOperation`&#160;&lt;a href=&quot;https://github.com/mongodb/mongo/blob/r4.4.3/src/mongo/db/service_context.cpp#L370&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/blob/r4.4.3/src/mongo/db/service_context.cpp#L370&lt;/a&gt;&lt;/li&gt;
	&lt;li&gt;and it has case when `resetOperationContext` is not called because opCtx Id is not found in `_clientByOperationId`&#160;&lt;a href=&quot;https://github.com/mongodb/mongo/blob/r4.4.3/src/mongo/db/service_context.cpp#L374-L380&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/blob/r4.4.3/src/mongo/db/service_context.cpp#L374-L380&lt;/a&gt;&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;I believe it is because:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;`OperationID` is just `uint32_t`&#160;&lt;a href=&quot;https://github.com/mongodb/mongo/blob/r4.4.3/src/mongo/db/service_context.h#L131&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/blob/r4.4.3/src/mongo/db/service_context.h#L131&lt;/a&gt;&lt;/li&gt;
	&lt;li&gt;And `_nextOpId` is of `OperationID` type&#160;&lt;a href=&quot;https://github.com/mongodb/mongo/blob/r4.4.3/src/mongo/db/service_context.h#L693&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/blob/r4.4.3/src/mongo/db/service_context.h#L693&lt;/a&gt;&lt;/li&gt;
	&lt;li&gt;it is just atomically incremented in `ServiceContext::makeOperationContext`&#160;&lt;a href=&quot;https://github.com/mongodb/mongo/blob/r4.4.3/src/mongo/db/service_context.cpp#L251&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/blob/r4.4.3/src/mongo/db/service_context.cpp#L251&lt;/a&gt;&lt;/li&gt;
	&lt;li&gt;and emplaced into `_clientByOperationId` without any check &lt;a href=&quot;https://github.com/mongodb/mongo/blob/r4.4.3/src/mongo/db/service_context.cpp#L277&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/blob/r4.4.3/src/mongo/db/service_context.cpp#L277&lt;/a&gt;&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Therefore I believe long living process just got `_nextOpId` wraparound and operation in `_clientByOperationId` is simply overwritten with more short living operation and then deleted. Then `_delistOperation` believes operation is already cleaned, and doesn&apos;t call `resetOperationContext`.&lt;/p&gt;

&lt;p&gt;Simplest fix is to add check in `ServiceContext::makeOperationContext` for already existed key in `_clientByOperationId`. Otherwise `OperationID` type has to be changed to `uint64_t`.&lt;/p&gt;</comment>
                            <comment id="3645392" author="bigbourin@gmail.com" created="Wed, 3 Mar 2021 20:21:28 +0000"  >&lt;p&gt;Ok thanks, I uploaded the logs + diagnostic.data for my 3 nodes (called bhs, db3 and rbx) on &lt;a href=&quot;https://10gen-httpsupload.s3.amazonaws.com/upload_forms/a1afa8da-a745-4424-818e-88da78048993.html&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;this portal&lt;/a&gt; and I am following this ticket.&lt;br/&gt;
Let me know if I can help more!&lt;/p&gt;</comment>
                            <comment id="3645279" author="JIRAUSER1254107" created="Wed, 3 Mar 2021 19:46:22 +0000"  >&lt;p&gt;Thanks for the report! If you want to upload your log data to this ticket we&apos;ll take a look, however, we&apos;ve seen a few instances of this issue and logging has not proved fruitful yet. We have a number of folks actively engaged in reproducing the problem to generate a core dump so we can track down a root cause. Please keep an eye on this ticket for further information.&#160;&lt;/p&gt;</comment>
                            <comment id="3643746" author="bigbourin@gmail.com" created="Wed, 3 Mar 2021 08:33:42 +0000"  >&lt;p&gt;I just got one of these chain-failure (like in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-52735&quot; title=&quot;mongodb crash with &amp;quot;Invariant failure&amp;quot; opCtx != nullptr &amp;amp;&amp;amp; _opCtx == nullptr&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-52735&quot;&gt;&lt;del&gt;SERVER-52735&lt;/del&gt;&lt;/a&gt;) on 2 out 3 of my RS nodes, leaving the RS in a read-only state. In my case it happened after a short network interruption (planned maintenance) on one of them (the first primary).&lt;br/&gt;
I went like:&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;db3.updn.io (Primary, 4.4.1) notices network split (&quot;Can&apos;t see a majority of the set, relinquishing primary&quot;) and crash with the same Invariant failure (&quot;opCtx != nullptr &amp;amp;&amp;amp; _opCtx == nullptr&quot;)&lt;/li&gt;
	&lt;li&gt;rbx.updn.io (Secondary, 4.4.2) at the same time notices the network split, triggering an election and assuming primary role (&quot;Election succeeded, assuming primary role&quot;) then instantly crash with the same exception (at the exact same second as db3).&lt;/li&gt;
	&lt;li&gt;bhs.updn.io (Secondary, 4.4.1) meanwhile, notices both server down but after voting for rbx once, don&apos;t assume primary as it&apos;s the only one left and stayed alive.&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;Do you need my log samples and diagnostic data? where shall I upload them? Thanks!&lt;/p&gt;</comment>
                            <comment id="3553330" author="billy.donahue" created="Thu, 7 Jan 2021 18:55:35 +0000"  >&lt;p&gt;Maybe we can do something like what heap checkers do, and record some backtrace information. Have all calls to setOperationContext record a address-only backtrace. We will only need memory for one of these traces per Client. Later when a competing setOperationContext call comes in, we can log the saved backtrace along with the incoming backtrace. This might show us why they are in conflict.&lt;/p&gt;

&lt;p&gt;I don&apos;t think backporting &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-49468&quot; title=&quot;Invalidate previous OperationContext when a new OperationContext is created&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-49468&quot;&gt;&lt;del&gt;SERVER-49468&lt;/del&gt;&lt;/a&gt; will yield better information. The purpose there was merely to keep servers alive after they suffer this invariant violation. That may not be safe to do! It downgrades the servers response from process-fatal to merely operation-fatal. We don&apos;t know where the corruption is coming from, and it might not be local to the single operation.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                            <outwardlinks description="duplicates">
                                        <issuelink>
            <issuekey id="1719099">SERVER-57007</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is duplicated by">
                                        <issuelink>
            <issuekey id="1537998">SERVER-52735</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1592961">SERVER-53857</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1851980">SERVER-59350</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1886086">SERVER-60303</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="1407903">SERVER-49468</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1537998">SERVER-52735</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1592961">SERVER-53857</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1589068">SERVER-53747</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="1576335">SERVER-53482</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>24.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>3.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="18953"><![CDATA[v4.4]]></customfieldvalue>
    <customfieldvalue key="16775"><![CDATA[v4.2]]></customfieldvalue>
    <customfieldvalue key="15640"><![CDATA[v4.0]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[5002K00000sdg7QQAQ, 5002K00000tTuVnQAK, 5002K00000tU8NoQAK, 5002K00000tUVo0QAG, 5002K00000s1Kj2QAE, 5002K00000tW8wJQAS, 5002K00000tRrniQAC, 5002K00000tSTWXQA4, 5002K00000uGW6iQAG, 5002K00000vegi5QAA, 5002K00000wYGR0QAO, 5002K00000xDB3kQAG, 5002K00000uzvFCQAY, 5002K00000zBfEsQAK, 5002K000011eXixQAE, 5006R00001nKdmkQAC]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 4 Jan 2021 21:18:48 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        2 years, 46 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            2 years, 46 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>bigbourin@gmail.com</customfieldvalue>
            <customfieldvalue>rectalogic</customfieldvalue>
            <customfieldvalue>bozaro@gmail.com</customfieldvalue>
            <customfieldvalue>ben.caimano@mongodb.com</customfieldvalue>
            <customfieldvalue>billy.donahue@mongodb.com</customfieldvalue>
            <customfieldvalue>dan@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>bscrkmp@gmail.com</customfieldvalue>
            <customfieldvalue>matthew.tretin@mongodb.com</customfieldvalue>
            <customfieldvalue>sz</customfieldvalue>
            <customfieldvalue>tyler.seip@mongodb.com</customfieldvalue>
            <customfieldvalue>funny.falcon@gmail.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hynv1b:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr4bg7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="4502">Service Arch 2021-03-08</customfieldvalue>
    <customfieldvalue id="4503">Service Arch 2021-03-22</customfieldvalue>
    <customfieldvalue id="4504">Service Arch 2021-04-05</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10555" key="com.atlassian.jira.plugin.system.customfieldtypes:float">
                        <customfieldname>Story Points</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>5.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hynhan:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>