<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:15:23 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-47839] CSRS member fails to be killed on 4.4.0-rc3</title>
                <link>https://jira.mongodb.org/browse/SERVER-47839</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Running a sharded cluster on 4.4.0-rc3. I killed all 3 CSRS members by running a &lt;tt&gt;kill &amp;lt;pid&amp;gt;&lt;/tt&gt; command. The first and third members were successfully shut down, but the second member was not able to shut down. My random guess at the most relevant error is here:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{&quot;t&quot;:{&quot;$date&quot;:&quot;2020-04-28T14:26:47.203+00:00&quot;},&quot;s&quot;:&quot;W&quot;, &quot;c&quot;:&quot;STORAGE&quot;, &quot;id&quot;:20561,  &quot;ctx&quot;:&quot;SignalHandler&quot;,&quot;msg&quot;:&quot;Error stepping down in non-command initiated shutdown path&quot;,&quot;attr&quot;:{&quot;error&quot;:{&quot;code&quot;:189,&quot;codeName&quot;:&quot;PrimarySteppedDown&quot;,&quot;errmsg&quot;:&quot;While waiting for secondaries to catch up before stepping down, this node decided to step down for other reasons&quot;}}}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;Attaching the full logs. &quot;member2_9008&quot; is the process that failed to be killed.&lt;/p&gt;

&lt;p&gt;Note &amp;#8211; this doesn&apos;t happen every time, but I can reliably trigger again. &lt;/p&gt;

&lt;p&gt;Spoke to &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=judah.schvimer&quot; class=&quot;user-hover&quot; rel=&quot;judah.schvimer&quot;&gt;judah.schvimer&lt;/a&gt; and he recommended filing a bug directly. &lt;/p&gt;</description>
                <environment></environment>
        <key id="1333164">SERVER-47839</key>
            <summary>CSRS member fails to be killed on 4.4.0-rc3</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13203">Gone away</resolution>
                                        <assignee username="janna.golden@mongodb.com">Janna Golden</assignee>
                                    <reporter username="louisa.berger@mongodb.com">Louisa Berger</reporter>
                        <labels>
                    </labels>
                <created>Wed, 29 Apr 2020 18:45:44 +0000</created>
                <updated>Mon, 8 Jan 2024 15:23:10 +0000</updated>
                            <resolved>Fri, 8 May 2020 15:02:26 +0000</resolved>
                                                                    <component>Replication</component>
                                        <votes>0</votes>
                                    <watches>8</watches>
                                                                                                                <comments>
                            <comment id="3075914" author="janna.golden" created="Fri, 8 May 2020 15:02:26 +0000"  >&lt;p&gt;This was fixed by &lt;a href=&quot;https://github.com/mongodb/mongo/commit/dfaf95cc1b3c0388371874191e0be30545a7d4e8&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;this commit&lt;/a&gt; in 4.4.0-rc4.&lt;/p&gt;</comment>
                            <comment id="3075537" author="louisa.berger@10gen.com" created="Fri, 8 May 2020 12:25:18 +0000"  >&lt;p&gt;Sure, go ahead!&lt;/p&gt;</comment>
                            <comment id="3074528" author="janna.golden" created="Thu, 7 May 2020 18:59:21 +0000"  >&lt;p&gt;Yay! I can close this ticket if you feel comfortable doing so.&lt;/p&gt;</comment>
                            <comment id="3074453" author="louisa.berger@10gen.com" created="Thu, 7 May 2020 18:22:55 +0000"  >&lt;p&gt;Ran our tests on rc4 and it looks resolved!&lt;/p&gt;</comment>
                            <comment id="3072404" author="janna.golden" created="Wed, 6 May 2020 18:27:37 +0000"  >&lt;p&gt;I think this should be fixed in rc4. I have not been able to reproduce the issue, but looking at the logs I&apos;m pretty convinced that this is the same issue described in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-46841&quot; title=&quot;Make PeriodicRunner interrupt blocked operations on stop&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-46841&quot;&gt;&lt;del&gt;SERVER-46841&lt;/del&gt;&lt;/a&gt; which was backported to rc4. That ticket in particular discusses fixing an issue shutting down the logical session cache when it&apos;s refreshing, but I believe the same issue described can happen with the PeriodicShardedIndexConsistencyChecker. We see the PeriodicShardedIndexConsistencyChecker log &lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;| 2020-04-28T14:26:37.204+00:00 I  SHARDING 22049   [PeriodicShardedIndexConsistencyChecker] &quot;Checking consistency of sharded collection indexes across the cluster&quot; &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;just before the SignalHandler logs that it received a shutdown signal. The PeriodicShardedIndexConsistencyChecker &lt;a href=&quot;https://github.com/mongodb/mongo/blob/201b8eb58920634b4519a8d3ea9b4c8c022b0875/src/mongo/db/s/periodic_sharded_index_consistency_checker.cpp#L166-L172&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;runs an agg pipeline&lt;/a&gt; that is run on the ARS, so we can get stuck &lt;a href=&quot;https://github.com/mongodb/mongo/blob/201b8eb58920634b4519a8d3ea9b4c8c022b0875/src/mongo/s/async_requests_sender.cpp#L100&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;here&lt;/a&gt; when running &lt;a href=&quot;https://github.com/mongodb/mongo/blob/201b8eb58920634b4519a8d3ea9b4c8c022b0875/src/mongo/db/db.cpp#L1110&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;PeriodicShardedIndexConsistencyChecker::onShutDown()&lt;/a&gt; in the same way as the LSC.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=louisa.berger&quot; class=&quot;user-hover&quot; rel=&quot;louisa.berger&quot;&gt;louisa.berger&lt;/a&gt; rc4 is getting cut tonight, so I think it might be easiest to see if rc4 fixes the issues you&apos;re seeing. If it doesn&apos;t I&apos;m happy to continue to try to repro after.&lt;/p&gt;</comment>
                            <comment id="3062262" author="tess.avitabile" created="Thu, 30 Apr 2020 12:37:23 +0000"  >&lt;p&gt;We catch the error from the stepdown attempt&#160;&lt;a href=&quot;https://github.com/mongodb/mongo/blob/r4.4.0-rc3/src/mongo/db/db.cpp#L1080-L1085&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;here&lt;/a&gt; and continue.&lt;/p&gt;

&lt;p&gt;We can see that we proceed in shutting down the &lt;tt&gt;MirrorMaestro&lt;/tt&gt; &lt;a href=&quot;https://github.com/mongodb/mongo/blob/r4.4.0-rc3/src/mongo/db/db.cpp#L1093&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;here&lt;/a&gt; by this log line:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;| 2020-04-28T14:26:47.203+00:00 I  REPL     40447   [TopologyVersionObserver] &quot;Stopped {topologyVersionObserverName}&quot;,&quot;attr&quot;:{&quot;topologyVersionObserverName&quot;:&quot;TopologyVersionObserver&quot;}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;However, we never reach shutting down the &lt;tt&gt;TransportLayer&lt;/tt&gt;, since we don&apos;t see &lt;a href=&quot;https://github.com/mongodb/mongo/blob/r4.4.0-rc3/src/mongo/db/db.cpp#L1115-L1117&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;this log line&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;This means there&apos;s a hang somewhere in &lt;a href=&quot;https://github.com/mongodb/mongo/blob/r4.4.0-rc3/src/mongo/db/db.cpp#L1095-L1111&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;this section&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;Sending to the Sharding team to investigate.&lt;/p&gt;</comment>
                            <comment id="3061123" author="lingzhi.deng" created="Wed, 29 Apr 2020 19:01:01 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-47190&quot; title=&quot;Shutdown command with force:true should ignore all stepdown errors&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-47190&quot;&gt;&lt;del&gt;SERVER-47190&lt;/del&gt;&lt;/a&gt; has not been backported yet. And &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-47190&quot; title=&quot;Shutdown command with force:true should ignore all stepdown errors&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-47190&quot;&gt;&lt;del&gt;SERVER-47190&lt;/del&gt;&lt;/a&gt; was not expected to change the behaviors of signal-based shutdown.&lt;/p&gt;</comment>
                            <comment id="3061095" author="judah.schvimer" created="Wed, 29 Apr 2020 18:49:29 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=lingzhi.deng&quot; class=&quot;user-hover&quot; rel=&quot;lingzhi.deng&quot;&gt;lingzhi.deng&lt;/a&gt;, any idea if this is related to &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-47190&quot; title=&quot;Shutdown command with force:true should ignore all stepdown errors&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-47190&quot;&gt;&lt;del&gt;SERVER-47190&lt;/del&gt;&lt;/a&gt;?&lt;br/&gt;
&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=mira.carey%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;mira.carey@mongodb.com&quot;&gt;mira.carey@mongodb.com&lt;/a&gt;, am I wrong that this is unexpected? Does this sound related to any of Service Arch&apos;s recent shutdown work?&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="258048" name="member1_9007.log" size="27928076" author="louisa.berger@mongodb.com" created="Wed, 29 Apr 2020 18:45:43 +0000"/>
                            <attachment id="258047" name="member2_9008.log" size="14896548" author="louisa.berger@mongodb.com" created="Wed, 29 Apr 2020 18:45:37 +0000"/>
                            <attachment id="258046" name="member3_9009.log" size="1203965" author="louisa.berger@mongodb.com" created="Wed, 29 Apr 2020 18:45:30 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>8.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 29 Apr 2020 18:49:29 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        3 years, 39 weeks, 5 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            3 years, 39 weeks, 5 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>janna.golden@mongodb.com</customfieldvalue>
            <customfieldvalue>judah.schvimer@mongodb.com</customfieldvalue>
            <customfieldvalue>lingzhi.deng@mongodb.com</customfieldvalue>
            <customfieldvalue>louisa.berger@mongodb.com</customfieldvalue>
            <customfieldvalue>tess.avitabile@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hxidvb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hw3i6f:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="3842">Sharding 2020-05-18</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hxi04n:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>