<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:10:53 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-6151] mongos crashed : corrupted unsorted chunks</title>
                <link>https://jira.mongodb.org/browse/SERVER-6151</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Mongos crashed.&lt;br/&gt;
Log extact :&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;
	&lt;ul&gt;
		&lt;li&gt;
		&lt;ul&gt;
			&lt;li&gt;glibc detected *** /usr/bin/mongos: free(): corrupted unsorted chunks: 0x0000000018ad2220 ***&lt;br/&gt;
Received signal 11&lt;br/&gt;
Backtrace: *** glibc detected *** /usr/bin/mongos: free(): corrupted unsorted chunks: 0x0000000018ad4520 ***&lt;br/&gt;
Logstream::get called in uninitialized state&lt;br/&gt;
Thu Jun 21 10:32:17 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn264410&amp;#93;&lt;/span&gt;   Assertion failure ! inShutdown() client/connpool.cpp 136&lt;/li&gt;
			&lt;li&gt;glibc detected *** /usr/bin/mongos: free(): corrupted unsorted chunks: 0x0000000018ab03f0 ***&lt;/li&gt;
			&lt;li&gt;glibc detected *** /usr/bin/mongos: double free or corruption (fasttop): 0x0000000018aaf7d0 ***&lt;/li&gt;
			&lt;li&gt;glibc detected *** /usr/bin/mongos: double free or corruption (fasttop): 0x0000000018aafab0 ***&lt;/li&gt;
			&lt;li&gt;glibc detected *** /usr/bin/mongos: double free or corruption (fasttop): 0x0000000018aafab0 ***&lt;/li&gt;
			&lt;li&gt;glibc detected *** /usr/bin/mongos: double free or corruption (fasttop): 0x0000000018aafab0 ***&lt;/li&gt;
			&lt;li&gt;glibc detected *** /usr/bin/mongos: double free or corruption (fasttop): 0x0000000018aafab0 ***&lt;/li&gt;
			&lt;li&gt;glibc detected *** /usr/bin/mongos: free(): corrupted unsorted chunks: 0x0000000018e6f540 ***&lt;/li&gt;
			&lt;li&gt;glibc detected *** /usr/bin/mongos: double free or corruption (fasttop): 0x0000000018aafab0 ***&lt;/li&gt;
			&lt;li&gt;glibc detected *** /usr/bin/mongos: double free or corruption (fasttop): 0x0000000018aafab0 ***&lt;/li&gt;
			&lt;li&gt;glibc detected *** /usr/bin/mongos: double free or corruption (fasttop): 0x0000000018aafab0 ***&lt;/li&gt;
			&lt;li&gt;glibc detected *** /usr/bin/mongos: double free or corruption (fasttop): 0x0000000018aafab0 ***&lt;/li&gt;
			&lt;li&gt;glibc detected *** /usr/bin/mongos: double free or corruption (fasttop): 0x0000000018aafab0 ***&lt;/li&gt;
			&lt;li&gt;glibc detected *** /usr/bin/mongos: double free or corruption (fasttop): 0x0000000018aafab0 ***&lt;/li&gt;
		&lt;/ul&gt;
		&lt;/li&gt;
	&lt;/ul&gt;
	&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;As seen on mms, mongos process used a lot of virtual memory (18Gb) before failure.&lt;/p&gt;</description>
                <environment>linux x86_64 2.6.18-274.3.1.el5.centos.plus</environment>
        <key id="41979">SERVER-6151</key>
            <summary>mongos crashed : corrupted unsorted chunks</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="4">Incomplete</resolution>
                                        <assignee username="greg_10gen">Greg Studer</assignee>
                                    <reporter username="khodin">Kl&#233;bert Hodin</reporter>
                        <labels>
                            <label>mongos</label>
                    </labels>
                <created>Thu, 21 Jun 2012 10:09:59 +0000</created>
                <updated>Tue, 16 Nov 2021 13:41:10 +0000</updated>
                            <resolved>Mon, 1 Oct 2012 16:53:37 +0000</resolved>
                                    <version>2.0.6</version>
                                                    <component>Stability</component>
                                        <votes>1</votes>
                                    <watches>4</watches>
                                                                                                                <comments>
                            <comment id="169883" author="greg_10gen" created="Mon, 1 Oct 2012 16:53:37 +0000"  >&lt;p&gt;Original submitter went away - second issue unrelated.&lt;/p&gt;</comment>
                            <comment id="157554" author="greg_10gen" created="Tue, 28 Aug 2012 15:54:00 +0000"  >&lt;p&gt;The previous issue was related to mongos - it seems like you&apos;re experiencing a problem with mongod (which could be related, but I&apos;m guessing is unlikely).  This problem seems more related to &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-2652&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;https://jira.mongodb.org/browse/SERVER-2652&lt;/a&gt;.&lt;/p&gt;</comment>
                            <comment id="156899" author="dg@doodle.com" created="Mon, 27 Aug 2012 10:34:00 +0000"  >&lt;p&gt;I&apos;m having similar issues (although I cannot tell if they&apos;re actually the same) with 2.0.7. The error occured when I tried to shut down mongodb (/etc/init.d/mongodb stop). See attached log file (mongodb.filtered.log). I have removed connect/disconnect/auth log statements.&lt;/p&gt;

&lt;p&gt;Environment is Debian squeeze/2.6.39 from backports.&lt;/p&gt;</comment>
                            <comment id="141416" author="greg_10gen" created="Tue, 10 Jul 2012 17:21:35 +0000"  >&lt;p&gt;Note - the above tools would require a debug build of mongos for the trace to be usable.  &lt;/p&gt;</comment>
                            <comment id="140449" author="greg_10gen" created="Fri, 6 Jul 2012 19:11:26 +0000"  >&lt;p&gt;Unfortunately no - we&apos;re working on recording log info but aren&apos;t there yet.  The logs would have helped us potentially pull out anything that seemed abnormal.&lt;/p&gt;

&lt;p&gt;Is it possible on your end to run memory usage tools on the mongos while running?  It would be extremely useful to get memory profiling information, one option is to use tcmalloc as doc&apos;d here:&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;http://gperftools.googlecode.com/svn/trunk/doc/heapprofile.html&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://gperftools.googlecode.com/svn/trunk/doc/heapprofile.html&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;The resulting output would target the portion of the mongos codebase that was allocating the memory.  This will have a performance impact, though it may be manageable if you&apos;re running background hadoop processes.&lt;/p&gt;</comment>
                            <comment id="140241" author="khodin" created="Fri, 6 Jul 2012 08:15:23 +0000"  >&lt;p&gt;No we are not. This period is 7 days long.&lt;/p&gt;

&lt;p&gt;What kind of infos do you need ? Startup logs ?&lt;br/&gt;
Is there a way to get it on our MMS dashboard ?&lt;/p&gt;</comment>
                            <comment id="139906" author="greg_10gen" created="Thu, 5 Jul 2012 15:39:37 +0000"  >&lt;p&gt;Thanks for the additional information - discussing with mongo-hadoop maintainers now.  From the previous crash, it appears you ran out of file descriptors on your system (which shouldn&apos;t have caused a crash, but indicates that you (or mongos) were using an unexpected number of connections).  &lt;/p&gt;

&lt;p&gt;Are you able to post the mongos log for a full period from startup to crash (feel free to post a SUPPORT ticket if you need to keep the log private)?&lt;/p&gt;</comment>
                            <comment id="139534" author="kamaradclimber" created="Wed, 4 Jul 2012 09:37:02 +0000"  >&lt;p&gt;some precisions : &lt;/p&gt;

&lt;p&gt;we run mongo-hadoop to dump around a 70GB collection on hdfs. The job gets around 100 map at the same time. Since it does not support connecting to multiple mongoS, one of them get all the load (100 concurrent chunks retrieved). Chunk size is fixed to 64mb.&lt;/p&gt;

&lt;p&gt;Memory used by the mongoS increases a lot until crash.&lt;br/&gt;
You can see on the following mms graph : &lt;a href=&quot;https://mms.10gen.com/chart/bookmark/4ff40250e4b02c1fc1aeaa6c&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://mms.10gen.com/chart/bookmark/4ff40250e4b02c1fc1aeaa6c&lt;/a&gt; that since we have started to use mongo-hadoop (~june 13th), memory has greatly increased.&lt;br/&gt;
The graph from this night : &lt;a href=&quot;https://mms.10gen.com/chart/bookmark/4ff40b58e4b0f1cff8285fed&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://mms.10gen.com/chart/bookmark/4ff40b58e4b0f1cff8285fed&lt;/a&gt; also show that there is a slow decrease at some point which lead to unresponsive mongoS instance.&lt;/p&gt;

</comment>
                            <comment id="139513" author="khodin" created="Wed, 4 Jul 2012 08:27:13 +0000"  >&lt;p&gt;Any updates on this issue ?&lt;/p&gt;

&lt;p&gt;We found out this crash occurs when running mongo-hadoop adapter. &lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo-hadoop&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo-hadoop&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="135286" author="khodin" created="Thu, 21 Jun 2012 13:32:55 +0000"  >&lt;p&gt;We&apos;re using 2.5 version.&lt;/p&gt;</comment>
                            <comment id="135282" author="scotthernandez" created="Thu, 21 Jun 2012 13:22:35 +0000"  >&lt;p&gt;What glibc version are you using?&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="18979" name="mongodb.filtered.log" size="21290" author="davidgubler" created="Mon, 27 Aug 2012 10:34:00 +0000"/>
                            <attachment id="17468" name="mongos_error_extract.log" size="68926" author="khodin" created="Thu, 21 Jun 2012 10:09:59 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>11.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Thu, 21 Jun 2012 13:22:35 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        11 years, 20 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            11 years, 20 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10000" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Old_Backport</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10000"><![CDATA[No]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10020"><![CDATA[Linux]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>dg@doodle.com</customfieldvalue>
            <customfieldvalue>kamaradclimber</customfieldvalue>
            <customfieldvalue>greg_10gen</customfieldvalue>
            <customfieldvalue>khodin</customfieldvalue>
            <customfieldvalue>scotthernandez</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrnzcv:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrgkof:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>10305</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|ht05pj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>