<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:04:17 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-3869] mongod 2.0.0 crash </title>
                <link>https://jira.mongodb.org/browse/SERVER-3869</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;The mongodb primary server in a replicaset crashed after running for hours,&lt;/p&gt;

&lt;p&gt;Here is the log:&lt;/p&gt;


&lt;p&gt;Thu Sep 15 19:33:57 Invalid access at address: 0x7fd8469db000&lt;/p&gt;

&lt;p&gt;Thu Sep 15 19:33:57 Got signal: 11 (Segmentation fault).&lt;/p&gt;

&lt;p&gt;Thu Sep 15 19:33:57 Backtrace:&lt;br/&gt;
0xa83fc9 0xa845a0 0x7fe935a718f0 0x7fe934fb80ba 0x909ee2 0x95e73c 0x880494 0x88526a 0xa96a46 0x635dd7 0x7fe935a689ca 0x7fe93501770d&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo10abruptQuitEi+0x399) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa83fc9&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo24abruptQuitWithAddrSignalEiP7siginfoPv+0x220) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa845a0&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/libpthread.so.0(+0xf8f0) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fe935a718f0&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/libc.so.6(memcpy+0x42a) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fe934fb80ba&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo22fillQueryResultFromObjERNS_11_BufBuilderINS_16TrivialAllocatorEEEPNS_10ProjectionERKNS_7BSONObjEPNS_7DiskLocE+0x8c2) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x909ee2&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo14processGetMoreEPKcixRNS_5CurOpEiRb+0x86c) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x95e73c&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo15receivedGetMoreERNS_10DbResponseERNS_7MessageERNS_5CurOpE+0x1b4) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x880494&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo16assembleResponseERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xf6a) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x88526a&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0x76) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa96a46&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo3pms9threadRunEPNS_13MessagingPortE+0x287) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x635dd7&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/libpthread.so.0(+0x69ca) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fe935a689ca&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/libc.so.6(clone+0x6d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fe93501770d&amp;#93;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;Thu Sep 15 19:33:57 dbexit:&lt;br/&gt;
Thu Sep 15 19:33:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn36194&amp;#93;&lt;/span&gt; shutdown: going to close listening sockets...&lt;br/&gt;
Thu Sep 15 19:33:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn36194&amp;#93;&lt;/span&gt; closing listening socket: 5&lt;br/&gt;
Thu Sep 15 19:33:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn36194&amp;#93;&lt;/span&gt; closing listening socket: 6&lt;br/&gt;
Thu Sep 15 19:33:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn36194&amp;#93;&lt;/span&gt; closing listening socket: 7&lt;br/&gt;
Thu Sep 15 19:33:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn36194&amp;#93;&lt;/span&gt; removing socket file: /tmp/mongodb-27017.sock&lt;br/&gt;
Thu Sep 15 19:33:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn36194&amp;#93;&lt;/span&gt; shutdown: going to flush diaglog...&lt;br/&gt;
Thu Sep 15 19:33:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn36194&amp;#93;&lt;/span&gt; shutdown: going to close sockets...&lt;br/&gt;
Thu Sep 15 19:33:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn36194&amp;#93;&lt;/span&gt; shutdown: waiting for fs preallocator...&lt;br/&gt;
Thu Sep 15 19:33:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn36194&amp;#93;&lt;/span&gt; shutdown: closing all files...&lt;br/&gt;
Thu Sep 15 19:33:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn1&amp;#93;&lt;/span&gt; end connection 127.0.0.1:59305&lt;br/&gt;
Thu Sep 15 19:33:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn36224&amp;#93;&lt;/span&gt; got request after shutdown()&lt;br/&gt;
Thu Sep 15 19:33:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn38857&amp;#93;&lt;/span&gt; got request after shutdown()&lt;br/&gt;
Thu Sep 15 19:33:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn32693&amp;#93;&lt;/span&gt; got request after shutdown()&lt;br/&gt;
Thu Sep 15 19:33:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn36192&amp;#93;&lt;/span&gt; got request after shutdown()&lt;br/&gt;
Thu Sep 15 19:33:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn36202&amp;#93;&lt;/span&gt; got request after shutdown()&lt;br/&gt;
Thu Sep 15 19:33:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn36198&amp;#93;&lt;/span&gt; got request after shutdown()&lt;br/&gt;
Thu Sep 15 19:33:57 Invalid access at address: 0&lt;/p&gt;

&lt;p&gt;Thu Sep 15 19:33:57 Got signal: 11 (Segmentation fault).&lt;/p&gt;

&lt;p&gt;Thu Sep 15 19:33:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn32527&amp;#93;&lt;/span&gt; got request after shutdown()&lt;br/&gt;
Thu Sep 15 19:33:58 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn39067&amp;#93;&lt;/span&gt; got request after shutdown()&lt;br/&gt;
Thu Sep 15 19:33:58 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn36218&amp;#93;&lt;/span&gt; got request after shutdown()&lt;br/&gt;
Thu Sep 15 19:33:58 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn32643&amp;#93;&lt;/span&gt; got request after shutdown()&lt;br/&gt;
Thu Sep 15 19:33:58 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn32668&amp;#93;&lt;/span&gt; got request after shutdown()&lt;br/&gt;
Thu Sep 15 19:33:58 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn39312&amp;#93;&lt;/span&gt; got request after shutdown()&lt;br/&gt;
Thu Sep 15 19:33:58 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn38971&amp;#93;&lt;/span&gt; got request after shutdown()&lt;br/&gt;
Thu Sep 15 19:33:58 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn33274&amp;#93;&lt;/span&gt; got request after shutdown()&lt;br/&gt;
Thu Sep 15 19:33:58 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn36200&amp;#93;&lt;/span&gt; got request after shutdown()&lt;br/&gt;
Thu Sep 15 19:33:58 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn33230&amp;#93;&lt;/span&gt; got request after shutdown()&lt;br/&gt;
Thu Sep 15 19:33:58 ERROR: Client::shutdown not called: slaveTracking&lt;br/&gt;
Thu Sep 15 19:33:58 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn36210&amp;#93;&lt;/span&gt; got request after shutdown()&lt;br/&gt;
Thu Sep 15 19:33:58 Backtrace:&lt;br/&gt;
0xa83fc9 0xa845a0 0x7fe935a718f0 0x7fe934fab392 0x7fe934faee83 0xa9d8e1 0x7fe935a68a61 0x7fe93501770d&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo10abruptQuitEi+0x399) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa83fc9&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo24abruptQuitWithAddrSignalEiP7siginfoPv+0x220) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa845a0&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/libpthread.so.0(+0xf8f0) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fe935a718f0&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/libc.so.6(+0x7a392) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fe934fab392&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/libc.so.6(cfree+0x73) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fe934faee83&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(tls_destructor+0xb1) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa9d8e1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/libpthread.so.0(+0x6a61) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fe935a68a61&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/libc.so.6(clone+0x6d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fe93501770d&amp;#93;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;Thu Sep 15 19:33:58 dbexit: ; exiting immediately&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;
	&lt;ul&gt;
		&lt;li&gt;
		&lt;ul&gt;
			&lt;li&gt;glibc detected *** /usr/bin/mongod: double free or corruption (fasttop): 0x0000000003688700 ***&lt;br/&gt;
======= Backtrace: =========&lt;br/&gt;
/lib/libc.so.6(+0x775b6)&lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fe934fa85b6&amp;#93;&lt;/span&gt;&lt;br/&gt;
/lib/libc.so.6(cfree+0x73)&lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fe934faee83&amp;#93;&lt;/span&gt;&lt;br/&gt;
/usr/bin/mongod(&lt;em&gt;ZNSt8_Rb_treeIPN5mongo9MongoFileES2_St9_IdentityIS2_ESt4lessIS2_ESaIS2_EE5eraseERKS2&lt;/em&gt;+0x144)&lt;span class=&quot;error&quot;&gt;&amp;#91;0x67b7b4&amp;#93;&lt;/span&gt;&lt;br/&gt;
/usr/bin/mongod(_ZN5mongo9MongoFile9destroyedEv+0x2c)&lt;span class=&quot;error&quot;&gt;&amp;#91;0x67a59c&amp;#93;&lt;/span&gt;&lt;br/&gt;
/usr/bin/mongod(_ZN5mongo8MongoMMF5closeEv+0x59)&lt;span class=&quot;error&quot;&gt;&amp;#91;0x759149&amp;#93;&lt;/span&gt;&lt;br/&gt;
/usr/bin/mongod(_ZN5mongo9MongoFile13closeAllFilesERSt18basic_stringstreamIcSt11char_traitsIcESaIcEE+0x151)&lt;span class=&quot;error&quot;&gt;&amp;#91;0x67b1c1&amp;#93;&lt;/span&gt;&lt;br/&gt;
/usr/bin/mongod(_ZN5mongo14shutdownServerEv+0x6d1)&lt;span class=&quot;error&quot;&gt;&amp;#91;0x881071&amp;#93;&lt;/span&gt;&lt;br/&gt;
/usr/bin/mongod(_ZN5mongo6dbexitENS_8ExitCodeEPKcb+0x2a0)&lt;span class=&quot;error&quot;&gt;&amp;#91;0x881710&amp;#93;&lt;/span&gt;&lt;br/&gt;
/usr/bin/mongod(_ZN5mongo10abruptQuitEi+0x4f6)&lt;span class=&quot;error&quot;&gt;&amp;#91;0xa84126&amp;#93;&lt;/span&gt;&lt;br/&gt;
/usr/bin/mongod(_ZN5mongo24abruptQuitWithAddrSignalEiP7siginfoPv+0x220)&lt;span class=&quot;error&quot;&gt;&amp;#91;0xa845a0&amp;#93;&lt;/span&gt;&lt;br/&gt;
/lib/libpthread.so.0(+0xf8f0)&lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fe935a718f0&amp;#93;&lt;/span&gt;&lt;br/&gt;
/lib/libc.so.6(memcpy+0x42a)&lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fe934fb80ba&amp;#93;&lt;/span&gt;&lt;br/&gt;
/usr/bin/mongod(_ZN5mongo22fillQueryResultFromObjERNS_11_BufBuilderINS_16TrivialAllocatorEEEPNS_10ProjectionERKNS_7BSONObjEPNS_7DiskLocE+0x8c2)&lt;span class=&quot;error&quot;&gt;&amp;#91;0x909ee2&amp;#93;&lt;/span&gt;&lt;br/&gt;
/usr/bin/mongod(_ZN5mongo14processGetMoreEPKcixRNS_5CurOpEiRb+0x86c)&lt;span class=&quot;error&quot;&gt;&amp;#91;0x95e73c&amp;#93;&lt;/span&gt;&lt;br/&gt;
/usr/bin/mongod(_ZN5mongo15receivedGetMoreERNS_10DbResponseERNS_7MessageERNS_5CurOpE+0x1b4)&lt;span class=&quot;error&quot;&gt;&amp;#91;0x880494&amp;#93;&lt;/span&gt;&lt;br/&gt;
/usr/bin/mongod(_ZN5mongo16assembleResponseERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xf6a)&lt;span class=&quot;error&quot;&gt;&amp;#91;0x88526a&amp;#93;&lt;/span&gt;&lt;br/&gt;
/usr/bin/mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0x76)&lt;span class=&quot;error&quot;&gt;&amp;#91;0xa96a46&amp;#93;&lt;/span&gt;&lt;br/&gt;
/usr/bin/mongod(_ZN5mongo3pms9threadRunEPNS_13MessagingPortE+0x287)&lt;span class=&quot;error&quot;&gt;&amp;#91;0x635dd7&amp;#93;&lt;/span&gt;&lt;br/&gt;
/lib/libpthread.so.0(+0x69ca)&lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fe935a689ca&amp;#93;&lt;/span&gt;&lt;br/&gt;
/lib/libc.so.6(clone+0x6d)&lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fe93501770d&amp;#93;&lt;/span&gt;&lt;/li&gt;
		&lt;/ul&gt;
		&lt;/li&gt;
	&lt;/ul&gt;
	&lt;/li&gt;
&lt;/ul&gt;
</description>
                <environment>Ubuntu 10.04.2, &lt;a href=&quot;http://downloads-distro.mongodb.org/repo/ubuntu-upstart/dists/dist/10gen/binary-amd64/mongodb-10gen_2.0.0_amd64.deb&quot;&gt;http://downloads-distro.mongodb.org/repo/ubuntu-upstart/dists/dist/10gen/binary-amd64/mongodb-10gen_2.0.0_amd64.deb&lt;/a&gt;</environment>
        <key id="22381">SERVER-3869</key>
            <summary>mongod 2.0.0 crash </summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="3">Duplicate</resolution>
                                        <assignee username="aaron">Aaron Staple</assignee>
                                    <reporter username="lqs">Liu Qishuai</reporter>
                        <labels>
                            <label>FRVIa</label>
                    </labels>
                <created>Thu, 15 Sep 2011 11:50:56 +0000</created>
                <updated>Thu, 6 Apr 2023 18:54:32 +0000</updated>
                            <resolved>Sun, 16 Oct 2011 22:14:37 +0000</resolved>
                                    <version>2.0.0</version>
                                                    <component>Index Maintenance</component>
                    <component>Querying</component>
                                        <votes>4</votes>
                                    <watches>8</watches>
                                                                                                                <comments>
                            <comment id="60791" author="eliot" created="Sun, 16 Oct 2011 22:14:37 +0000"  >&lt;p&gt;See &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-3961&quot; title=&quot;Segmentation Fault when creating new indexes&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-3961&quot;&gt;&lt;del&gt;SERVER-3961&lt;/del&gt;&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="59742" author="scotthernandez" created="Tue, 11 Oct 2011 13:00:45 +0000"  >&lt;p&gt;Another workaround for the geoquery getmore (cursor bug) issue is to use a limit which won&apos;t require using a cursor, or sending additional batches. Something like limit(100) for example should eliminate the problem for testing. It is possible that not all you have the same issue, but if everyone is using geo-queries than that is most likely it.&lt;/p&gt;</comment>
                            <comment id="58956" author="christian.tonhaeuser@navteq.com" created="Thu, 6 Oct 2011 21:53:19 +0000"  >&lt;p&gt;I analyzed this problem together with Scott, and he found out that it&apos;s probably a bug in the spatial search code that may occur when concurrent reads and writes are executed on the same dataset.&lt;br/&gt;
We got rid of the spatial indexes in our data model and replaced them with something else and now the crash is gone.&lt;/p&gt;

&lt;p&gt;According to Scott, the error should be corrected in the current dev branch, but it might take a little longer until an updated build containing the fix is available.&lt;/p&gt;</comment>
                            <comment id="56875" author="christian.tonhaeuser@navteq.com" created="Tue, 27 Sep 2011 11:37:31 +0000"  >&lt;p&gt;@Eliot: Our current data model looks roughly like this:&lt;br/&gt;
{&lt;br/&gt;
    &quot;_id&quot;: NumberLong(123456789),&lt;br/&gt;
    &quot;instanceId&quot;: &quot;Test&quot;,&lt;br/&gt;
    &quot;description&quot;: null,&lt;br/&gt;
    &quot;createdTS&quot;: NumberLong(&quot;1313506888027&quot;),&lt;br/&gt;
    &quot;lastUpdateTS&quot;: NumberLong(&quot;1316443719486&quot;),&lt;br/&gt;
    &quot;expiresTS&quot;: NumberLong(0),&lt;br/&gt;
    &quot;properties&quot;: &lt;/p&gt;
{
        &quot;p1&quot;: 5,
        &quot;p2&quot;: &quot;B&quot;,
        &quot;p3&quot;: &quot;&quot;,
        &quot;p4&quot;: &quot;&quot;,
        &quot;p5&quot;: &quot;&quot;,
        &quot;p6&quot;: 991,
        &quot;p7&quot;: 1,
        &quot;p8&quot;: 6,
        &quot;p9&quot;: &quot;&quot;,
        &quot;p10&quot;: &quot;0:0-0&quot;,
        &quot;p11&quot;: 0,
        &quot;p12&quot;: 1,
        &quot;p13&quot;: 0,
        &quot;p14&quot;: &quot;&quot;,
        &quot;p15&quot;: &quot;-&quot;,
        &quot;p16&quot;: &quot;-&quot;,
        &quot;p17&quot;: [
            &quot;MYS&quot;
        ]
    }
&lt;p&gt;,&lt;br/&gt;
    &quot;version&quot;: 0,&lt;br/&gt;
    &quot;binData&quot;: null,&lt;br/&gt;
    &quot;type&quot;: &quot;Line&quot;,&lt;br/&gt;
    &quot;coordinates&quot;: [ &lt;span class=&quot;error&quot;&gt;&amp;#91;114.00405, 4.41043, 0&amp;#93;&lt;/span&gt;, [ 114.00463, 4.41081, 0]],&lt;br/&gt;
    &quot;bbox&quot;: [[ 114.00405, 4.41043], [ 114.00463, 4.41081]]&lt;br/&gt;
}&lt;/p&gt;

&lt;p&gt;(Note: The amount of coordinates in the &quot;coordinates&quot; array is usually between 2 and 8 entries for most objects. However, it is not bounded by our application, there are objects with &amp;gt; 50 coordinate values in the array)&lt;/p&gt;

&lt;p&gt;Currently, the only indexes on the collection are on the _id field (which is also used for sharding) and on the &quot;bbox&quot; array, which is a spatial index. (Note: The bbox field came to pass when we noticed the first crashes. We still had the spatial index on the coordinates array then and therefore tried to reduce the size of the index by simply storing the spatial bounding box of the objects. Didn&apos;t help, though...)&lt;/p&gt;

&lt;p&gt;The only requests we are doing are rectangle ($within) queries on the bbox field for reading and &quot;fake&quot; updates, i.e. updates that don&apos;t really change the object, but trigger a write operation on the DB anyway.&lt;/p&gt;

&lt;p&gt;Our biggest colletion contains ~124 million objects at the moment, but it will slowly grow in the future.&lt;/p&gt;

&lt;p&gt;When running load tests with around 50-100 simulated users that do about 98% read operations and 2% write operations (with 1-5 objects contained in every write), it takes at most half an hour until one of the primaries from our replica sets crashes.&lt;br/&gt;
When we are running lower loads (e.g. 10 concurrent users) everything is fine.&lt;/p&gt;

&lt;p&gt;Also, when running read-only or write-only tests everything&apos;s fine, too.&lt;/p&gt;</comment>
                            <comment id="56618" author="dhoene" created="Mon, 26 Sep 2011 14:01:45 +0000"  >&lt;p&gt;@Christian:&lt;/p&gt;

&lt;p&gt;Thx, I filed another issue: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-3947&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;https://jira.mongodb.org/browse/SERVER-3947&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="56616" author="christian.tonhaeuser@navteq.com" created="Mon, 26 Sep 2011 13:47:00 +0000"  >&lt;p&gt;@Eliot: Please contact me by eMail so we can discuss what kind of data and code I might be able to provide.&lt;br/&gt;
Furthermore, we have conducted further testing on single instances, and it seems that with a write-only load the problem does not manifest itself. Will try that out on our sharded environment soon.&lt;/p&gt;


&lt;p&gt;@Dennis: This seems to be a different error from the one we excperience, at least the backtrace is radically different.&lt;/p&gt;
</comment>
                            <comment id="56611" author="dhoene" created="Mon, 26 Sep 2011 13:02:11 +0000"  >&lt;p&gt;I&apos;m getting a similar misbehaviour during heavy doc inputs and transformations with multiple connections on a &lt;/p&gt;


&lt;ol&gt;
	&lt;li&gt;uname -a&lt;br/&gt;
Linux winlager-eval 2.6.18-194.32.1.el5 #1 SMP Wed Jan 5 17:52:25 EST 2011 x86_64 x86_64 x86_64 GNU/Linux&lt;/li&gt;
	&lt;li&gt;cat /etc/redhat-release&lt;br/&gt;
CentOS release 5.5 (Final)&lt;/li&gt;
&lt;/ol&gt;



&lt;p&gt;I&apos;m running a single server instance 2.0.0, no replication/sharding. The problem occured since server update from 1.8 to 2.0 and php-driver update from 1.1.4 to 1.2.5. mongod terminates with the following log output:&lt;/p&gt;


&lt;p&gt;Mon Sep 26 14:52:22 Invalid access at address: 0&lt;/p&gt;

&lt;p&gt;Mon Sep 26 14:52:22 Got signal: 11 (Segmentation fault).&lt;/p&gt;

&lt;p&gt;Mon Sep 26 14:52:22 Backtrace:&lt;br/&gt;
0xa83fc9 0xa845a0 0x3a5ba0eb10 0x54d7dc 0x83d551 0x83d756 0x83d8b0 0x95e2e9 0x880494 0x88526a 0xa96a46 0x635dd7 0x3a5ba0673d 0x3a5b2d3f6d &lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo10abruptQuitEi+0x399) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa83fc9&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo24abruptQuitWithAddrSignalEiP7siginfoPv+0x220) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa845a0&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib64/libpthread.so.0 &lt;span class=&quot;error&quot;&gt;&amp;#91;0x3a5ba0eb10&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo24FieldRangeVectorIterator7advanceERKNS_7BSONObjE+0x4c) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x54d7dc&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo11BtreeCursor29skipOutOfRangeKeysAndCheckEndEv+0x81) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x83d551&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo11BtreeCursor12skipAndCheckEv+0x26) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x83d756&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo11BtreeCursor7advanceEv+0x100) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x83d8b0&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo14processGetMoreEPKcixRNS_5CurOpEiRb+0x419) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x95e2e9&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo15receivedGetMoreERNS_10DbResponseERNS_7MessageERNS_5CurOpE+0x1b4) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x880494&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo16assembleResponseERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xf6a) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x88526a&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0x76) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa96a46&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo3pms9threadRunEPNS_13MessagingPortE+0x287) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x635dd7&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib64/libpthread.so.0 &lt;span class=&quot;error&quot;&gt;&amp;#91;0x3a5ba0673d&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib64/libc.so.6(clone+0x6d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x3a5b2d3f6d&amp;#93;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;Logstream::get called in uninitialized state&lt;br/&gt;
Mon Sep 26 14:52:22 ERROR: Client::~Client _context should be null but is not; client:conn&lt;br/&gt;
Logstream::get called in uninitialized state&lt;br/&gt;
Mon Sep 26 14:52:22 ERROR: Client::shutdown not called: conn&lt;/p&gt;</comment>
                            <comment id="56155" author="eliot" created="Fri, 23 Sep 2011 14:17:15 +0000"  >&lt;p&gt;Can you send the queries or operations you are doing?&lt;br/&gt;
Would it be possible to get a copy of the data set with ops or code? (can be private)&lt;/p&gt;</comment>
                            <comment id="56139" author="christian.tonhaeuser@navteq.com" created="Fri, 23 Sep 2011 12:19:07 +0000"  >&lt;p&gt;We now found out that this issue will also crash single server instances, albeit not as frequently (or quickly) as with a ReplicaSet.&lt;/p&gt;

&lt;p&gt;Please raise the priority to blocking.&lt;/p&gt;</comment>
                            <comment id="55871" author="christian.tonhaeuser@navteq.com" created="Thu, 22 Sep 2011 13:17:17 +0000"  >&lt;p&gt;Turning off NUMA-support in the kernel does not help.&lt;br/&gt;
Also, the crash already happened back in some 1.9 nightly build that we still had around for some reason.&lt;/p&gt;

&lt;p&gt;The crash is triggered quite quickly when we&apos;re running load tests on our application.&lt;br/&gt;
Single server instances don&apos;t exhibit this behaviour.&lt;/p&gt;</comment>
                            <comment id="55857" author="christian.tonhaeuser@navteq.com" created="Thu, 22 Sep 2011 11:53:25 +0000"  >&lt;p&gt;We are encountering the exact same stacktrace on CentOS 5.6 64Bit using the prebuilt (non-static) binaries.&lt;/p&gt;

&lt;p&gt;We are running a sharded environment, using three shards, each consisting of primary, secondary and an arbiter.&lt;br/&gt;
The primary replset members seem to crash at random, i.e. it&apos;s always a different mongod process going down.&lt;br/&gt;
The whole cluster is running on six machines in total, including three configserver instances.&lt;/p&gt;

&lt;p&gt;Currently running NUMA-Kernel (2.6.18-238.el5), will test soon with a non-NUMA kernel (in case this helps...).&lt;/p&gt;

&lt;p&gt;If you need further logs or would like some tests to be executed just ask... &lt;img class=&quot;emoticon&quot; src=&quot;https://jira.mongodb.org/images/icons/emoticons/wink.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/p&gt;

&lt;p&gt;Backtrace of the crash:&lt;br/&gt;
Thu Sep 22 12:34:46 Got signal: 11 (Segmentation fault).&lt;/p&gt;

&lt;p&gt;Thu Sep 22 12:34:46 Backtrace:&lt;br/&gt;
0xa83fc9 0xa845a0 0x395540eb10 0x395487c651 0x909ee2 0x95e73c 0x880494 0x88526a 0xa96a46 0x635dd7 0x395540673d 0x39548d40cd &lt;br/&gt;
 /data/project/mongodb/bin/mongod(_ZN5mongo10abruptQuitEi+0x399) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa83fc9&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /data/project/mongodb/bin/mongod(_ZN5mongo24abruptQuitWithAddrSignalEiP7siginfoPv+0x220) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa845a0&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib64/libpthread.so.0 &lt;span class=&quot;error&quot;&gt;&amp;#91;0x395540eb10&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib64/libc.so.6(memcpy+0x411) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x395487c651&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /data/project/mongodb/bin/mongod(_ZN5mongo22fillQueryResultFromObjERNS_11_BufBuilderINS_16TrivialAllocatorEEEPNS_10ProjectionERKNS_7BSONObjEPNS_7DiskLocE+0x8c2) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x909ee2&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /data/project/mongodb/bin/mongod(_ZN5mongo14processGetMoreEPKcixRNS_5CurOpEiRb+0x86c) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x95e73c&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /data/project/mongodb/bin/mongod(_ZN5mongo15receivedGetMoreERNS_10DbResponseERNS_7MessageERNS_5CurOpE+0x1b4) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x880494&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /data/project/mongodb/bin/mongod(_ZN5mongo16assembleResponseERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xf6a) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x88526a&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /data/project/mongodb/bin/mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0x76) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa96a46&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /data/project/mongodb/bin/mongod(_ZN5mongo3pms9threadRunEPNS_13MessagingPortE+0x287) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x635dd7&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib64/libpthread.so.0 &lt;span class=&quot;error&quot;&gt;&amp;#91;0x395540673d&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib64/libc.so.6(clone+0x6d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x39548d40cd&amp;#93;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;Logstream::get called in uninitialized state&lt;br/&gt;
Thu Sep 22 12:34:46 ERROR: Client::~Client _context should be null but is not; client:conn&lt;br/&gt;
Logstream::get called in uninitialized state&lt;br/&gt;
Thu Sep 22 12:34:46 ERROR: Client::shutdown not called: conn&lt;/p&gt;</comment>
                            <comment id="54654" author="lqs" created="Fri, 16 Sep 2011 06:20:43 +0000"  >&lt;p&gt;more detailed log&lt;/p&gt;</comment>
                            <comment id="54641" author="eliot" created="Fri, 16 Sep 2011 05:21:47 +0000"  >&lt;p&gt;Can you send the full log?&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="13171" name="crash.log" size="59471" author="lqs" created="Fri, 16 Sep 2011 06:20:43 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>13.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 16 Sep 2011 05:21:47 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        12 years, 18 weeks, 3 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            12 years, 18 weeks, 3 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10000" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Old_Backport</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10000"><![CDATA[No]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10020"><![CDATA[Linux]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>aaron</customfieldvalue>
            <customfieldvalue>christian.tonhaeuser@navteq.com</customfieldvalue>
            <customfieldvalue>dhoene</customfieldvalue>
            <customfieldvalue>eliot</customfieldvalue>
            <customfieldvalue>lqs</customfieldvalue>
            <customfieldvalue>scotthernandez</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hroqi7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hriqev:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>22932</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrix7z:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>