<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:20:02 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-29152] Segfault in multiple shard primaries under regular load</title>
                <link>https://jira.mongodb.org/browse/SERVER-29152</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Our database is divided into 4 shards, each having one primary, secondary and arbiter. Primaries are r4.2xlarge servers on AWS EC2, and secondaries are r4.xlarge.&lt;/p&gt;

&lt;p&gt;Our work load is intensive in both reads and writes, but these servers usually handle the load without a problem. However during their regular work, primaries of 3 of the 4 shards suddenly crashed, within a very short time of each other. We don&apos;t know what could have caused this.&lt;/p&gt;

&lt;p&gt;Attached are the logs of the segfaults from the primary servers. The one from shard1 seems different that the other two.&lt;/p&gt;</description>
                <environment></environment>
        <key id="382893">SERVER-29152</key>
            <summary>Segfault in multiple shard primaries under regular load</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="samantha.ritter@mongodb.com">Samantha Ritter</assignee>
                                    <reporter username="meni">Meni Livne</reporter>
                        <labels>
                    </labels>
                <created>Fri, 12 May 2017 08:38:41 +0000</created>
                <updated>Mon, 30 Oct 2023 23:16:55 +0000</updated>
                            <resolved>Tue, 30 May 2017 17:49:11 +0000</resolved>
                                    <version>3.2.13</version>
                                    <fixVersion>3.2.14</fixVersion>
                    <fixVersion>3.4.5</fixVersion>
                    <fixVersion>3.5.9</fixVersion>
                                    <component>Networking</component>
                                        <votes>2</votes>
                                    <watches>15</watches>
                                                                                                                <comments>
                            <comment id="1582747" author="xgen-internal-githook" created="Tue, 30 May 2017 15:27:10 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;samantharitter&apos;, u&apos;name&apos;: u&apos;samantharitter&apos;, u&apos;email&apos;: u&apos;samantha.ritter@10gen.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-29152&quot; title=&quot;Segfault in multiple shard primaries under regular load&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-29152&quot;&gt;&lt;del&gt;SERVER-29152&lt;/del&gt;&lt;/a&gt; Do not cache logging ostream in threadlocal when in other thread-specific contexts&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/fad590916a30ff34dc8c3b37afcfffa2c4e5c8bc&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/fad590916a30ff34dc8c3b37afcfffa2c4e5c8bc&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1581172" author="samantha.ritter@10gen.com" created="Fri, 26 May 2017 19:19:32 +0000"  >&lt;p&gt;It appears our hook did not catch the 3.2 commit, it&apos;s here:&lt;/p&gt;

&lt;p&gt;Author: samantharitter&lt;br/&gt;
Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-29152&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;SERVER-29152&lt;/a&gt; Do not cache logging ostream in threadlocal when in other thread-specific contexts&lt;br/&gt;
Branch: v3.2&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/85aa900eae81fdc07d00aa4b1fb782f7ca5b4664&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/85aa900eae81fdc07d00aa4b1fb782f7ca5b4664&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1580831" author="xgen-internal-githook" created="Fri, 26 May 2017 14:33:08 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;samantharitter&apos;, u&apos;name&apos;: u&apos;samantharitter&apos;, u&apos;email&apos;: u&apos;samantha.ritter@10gen.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-29152&quot; title=&quot;Segfault in multiple shard primaries under regular load&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-29152&quot;&gt;&lt;del&gt;SERVER-29152&lt;/del&gt;&lt;/a&gt; Do not cache logging ostream in threadlocal when in other thread-specific contexts&lt;br/&gt;
Branch: v3.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/36a4a00321bb531190bcd00f523bce95a81b5ab2&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/36a4a00321bb531190bcd00f523bce95a81b5ab2&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1577178" author="samantha.ritter@10gen.com" created="Mon, 22 May 2017 21:15:53 +0000"  >&lt;p&gt;Hi Meni,&lt;/p&gt;

&lt;p&gt;I wanted to update you on the status of this bug.  New logging code that was added by &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-28760&quot; title=&quot;Add egress logging to legacy write path&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-28760&quot;&gt;&lt;del&gt;SERVER-28760&lt;/del&gt;&lt;/a&gt; tries to log while a thread is exiting, in which case the logging subsystem may already be destroyed.  The order in which these objects are destroyed seems quasi-random, depending on the build or on the system&apos;s memory allocation.  This influences whether these objects are destroyed peacefully or whether they are destroyed in a bad order that leads to a crash.  We are investigating exactly what determines the ordering of the destruction of these objects.  We are still working to reproduce the crash on our end as we investigate what the best fix will be.  Thank you for your patience.&lt;/p&gt;

&lt;p&gt;As to what actual event may have triggered the thread to exit here in your case, can you provide complete log files from these crashes?  The stack traces you&apos;ve linked have been very helpful, and it would also help us to see what the system was doing up until things went south.&lt;/p&gt;

&lt;p&gt;Thank you,&lt;br/&gt;
Samantha&lt;/p&gt;</comment>
                            <comment id="1570373" author="meni" created="Sat, 13 May 2017 07:02:17 +0000"  >&lt;p&gt;We&apos;re using the mongodb-org-server packages for ubuntu from the official mongodb repositories. As far as we know these don&apos;t add any log rotation settings, and we haven&apos;t implemented any ourselves, and never noticed the log file being rotated.&lt;/p&gt;</comment>
                            <comment id="1569963" author="samantha.ritter@10gen.com" created="Fri, 12 May 2017 18:31:41 +0000"  >&lt;p&gt;Hi there,&lt;/p&gt;

&lt;p&gt;Thanks for opening this ticket, I&apos;m sorry you experienced these crashes.  I&apos;m looking into what might have happened on these servers.  Given the stack traces, it&apos;s possible that we have a bug in our logging subsystem.  Are you running with rotating log files?  If so, is there any chance that these servers&apos; log files were being rotated around the time the crash occurred?&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Samantha&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                                                <inwardlinks description="is duplicated by">
                                        <issuelink>
            <issuekey id="391557">SERVER-29510</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="385473">SERVER-29310</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="387111">SERVER-29377</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="155875" name="shard0-primary.txt" size="186799" author="meni" created="Fri, 12 May 2017 08:30:58 +0000"/>
                            <attachment id="155874" name="shard1-primary.txt" size="3963" author="meni" created="Fri, 12 May 2017 08:32:16 +0000"/>
                            <attachment id="155876" name="shard2-primary.txt" size="278294" author="meni" created="Fri, 12 May 2017 08:30:58 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>6.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="14340"><![CDATA[v3.4]]></customfieldvalue>
    <customfieldvalue key="13440"><![CDATA[v3.2]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 12 May 2017 14:23:36 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        6 years, 37 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            6 years, 37 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>meni</customfieldvalue>
            <customfieldvalue>samantha.ritter@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|ht7bev:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hszhk7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hs7pev:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>