<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:06:38 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-4665] Querying against sparse index creates bad state, affecting later queries</title>
                <link>https://jira.mongodb.org/browse/SERVER-4665</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;I have run into a case where querying against a sparse index on a nested field puts the DB in a bad state that affects later queries on the same collection.  I originally discovered this problem using the ruby driver, but was able to reproduce the problem completely from the mongo console (see below).  I inserted comments, prefixed with #.  The mongo_bug db did not exist prior to this example.&lt;/p&gt;

&lt;p&gt;&amp;gt; use mongo_bug&lt;br/&gt;
switched to db mongo_bug&lt;br/&gt;
&amp;gt; db.bug_collection.ensureIndex(&lt;/p&gt;
{&quot;nested.key2&quot;: 1}
&lt;p&gt;, &lt;/p&gt;
{sparse: true}
&lt;p&gt;)&lt;br/&gt;
&amp;gt; db.bug_collection.insert(&lt;/p&gt;
{key: &apos;a&apos;}
&lt;p&gt;)&lt;br/&gt;
&amp;gt; db.bug_collection.insert(&lt;/p&gt;
{key: &apos;b&apos;}
&lt;p&gt;)&lt;br/&gt;
&amp;gt; db.bug_collection.insert(&lt;/p&gt;
{key: &apos;c&apos;}
&lt;p&gt;)&lt;br/&gt;
&amp;gt; db.bug_collection.find({})&lt;/p&gt;
{ &quot;_id&quot; : ObjectId(&quot;4f0e7daf9be3426d0ba2e7b8&quot;), &quot;key&quot; : &quot;a&quot; }
{ &quot;_id&quot; : ObjectId(&quot;4f0e7db59be3426d0ba2e7b9&quot;), &quot;key&quot; : &quot;b&quot; }
{ &quot;_id&quot; : ObjectId(&quot;4f0e7dba9be3426d0ba2e7ba&quot;), &quot;key&quot; : &quot;c&quot; }
&lt;p&gt;&amp;gt; db.bug_collection.find({key: {$in: &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;#39;a&amp;#39;, &amp;#39;b&amp;#39;, &amp;#39;c&amp;#39;&amp;#93;&lt;/span&gt;}, &quot;nested.key2&quot;: {$ne: &apos;d&apos;}})&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;No results returned (expected, because of the sparse index?)&lt;br/&gt;
&amp;gt; db.bug_collection.find({key: {$in: &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;#39;a&amp;#39;, &amp;#39;b&amp;#39;, &amp;#39;c&amp;#39;&amp;#93;&lt;/span&gt;}})&lt;/li&gt;
	&lt;li&gt;No results returned (NOT expected)&lt;br/&gt;
&amp;gt; db.bug_collection.dropIndex(&quot;nested.key2_1&quot;)
{ &quot;nIndexesWas&quot; : 2, &quot;ok&quot; : 1 }
&lt;p&gt;&amp;gt; db.bug_collection.find({key: {$in: &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;#39;a&amp;#39;, &amp;#39;b&amp;#39;, &amp;#39;c&amp;#39;&amp;#93;&lt;/span&gt;}})&lt;/p&gt;
{ &quot;_id&quot; : ObjectId(&quot;4f0e7daf9be3426d0ba2e7b8&quot;), &quot;key&quot; : &quot;a&quot; }
{ &quot;_id&quot; : ObjectId(&quot;4f0e7db59be3426d0ba2e7b9&quot;), &quot;key&quot; : &quot;b&quot; }
{ &quot;_id&quot; : ObjectId(&quot;4f0e7dba9be3426d0ba2e7ba&quot;), &quot;key&quot; : &quot;c&quot; }&lt;/li&gt;
	&lt;li&gt;Query now works, after the index has been dropped.&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;The order of the queries also matters: &lt;/p&gt;

&lt;p&gt;&amp;gt; db.bug_collection.dropIndex(&quot;nested.key2_1&quot;)&lt;/p&gt;
{ &quot;nIndexesWas&quot; : 2, &quot;ok&quot; : 1 }
&lt;p&gt;&amp;gt; db.bug_collection.ensureIndex(&lt;/p&gt;
{&quot;nested.key2&quot;: 1}
&lt;p&gt;, &lt;/p&gt;
{sparse: true}
&lt;p&gt;)&lt;br/&gt;
&amp;gt; db.bug_collection.find({key: {$in: &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;#39;a&amp;#39;, &amp;#39;b&amp;#39;, &amp;#39;c&amp;#39;&amp;#93;&lt;/span&gt;}})&lt;/p&gt;
{ &quot;_id&quot; : ObjectId(&quot;4f0e811f89fc95d0a1a0356b&quot;), &quot;key&quot; : &quot;a&quot; }
{ &quot;_id&quot; : ObjectId(&quot;4f0e812389fc95d0a1a0356c&quot;), &quot;key&quot; : &quot;b&quot; }
{ &quot;_id&quot; : ObjectId(&quot;4f0e812589fc95d0a1a0356d&quot;), &quot;key&quot; : &quot;c&quot; }
&lt;ol&gt;
	&lt;li&gt;Results returned (expected)&lt;br/&gt;
&amp;gt; db.bug_collection.find({key: {$in: &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;#39;a&amp;#39;, &amp;#39;b&amp;#39;, &amp;#39;c&amp;#39;&amp;#93;&lt;/span&gt;}, &quot;nested.key2&quot;: {$ne: &apos;d&apos;}})
{ &quot;_id&quot; : ObjectId(&quot;4f0e811f89fc95d0a1a0356b&quot;), &quot;key&quot; : &quot;a&quot; }
{ &quot;_id&quot; : ObjectId(&quot;4f0e812389fc95d0a1a0356c&quot;), &quot;key&quot; : &quot;b&quot; }
{ &quot;_id&quot; : ObjectId(&quot;4f0e812589fc95d0a1a0356d&quot;), &quot;key&quot; : &quot;c&quot; }&lt;/li&gt;
	&lt;li&gt;Results returned (this same query returns nothing in first example)&lt;br/&gt;
&amp;gt; db.bug_collection.dropIndex(&quot;nested.key2_1&quot;)
{ &quot;nIndexesWas&quot; : 2, &quot;ok&quot; : 1 }
&lt;p&gt;&amp;gt; db.bug_collection.ensureIndex(&lt;/p&gt;
{&quot;nested.key2&quot;: 1}
&lt;p&gt;, &lt;/p&gt;
{sparse: true}
&lt;p&gt;)&lt;br/&gt;
&amp;gt; db.bug_collection.find({key: {$in: &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;#39;a&amp;#39;, &amp;#39;b&amp;#39;, &amp;#39;c&amp;#39;&amp;#93;&lt;/span&gt;}, &quot;nested.key2&quot;: {$ne: &apos;d&apos;}})&lt;/p&gt;&lt;/li&gt;
	&lt;li&gt;No results returned (as in first example)&lt;br/&gt;
&amp;gt; db.bug_collection.find({key: {$in: &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;#39;a&amp;#39;, &amp;#39;b&amp;#39;, &amp;#39;c&amp;#39;&amp;#93;&lt;/span&gt;}})&lt;/li&gt;
	&lt;li&gt;No results returned (as in first example)&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;It&apos;s as if the queries are considered equivalent and being cached.&lt;/p&gt;</description>
                <environment>Shell and DB running under OSX.</environment>
        <key id="28185">SERVER-4665</key>
            <summary>Querying against sparse index creates bad state, affecting later queries</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="aaron">Aaron Staple</assignee>
                                    <reporter username="canavese">Paul Canavese</reporter>
                        <labels>
                    </labels>
                <created>Thu, 12 Jan 2012 06:54:59 +0000</created>
                <updated>Mon, 11 Jul 2016 18:33:43 +0000</updated>
                            <resolved>Tue, 17 Jan 2012 07:11:58 +0000</resolved>
                                    <version>2.0.1</version>
                                    <fixVersion>2.0.3</fixVersion>
                    <fixVersion>2.1.0</fixVersion>
                                    <component>Index Maintenance</component>
                    <component>Querying</component>
                                        <votes>2</votes>
                                    <watches>5</watches>
                                                                                                                <comments>
                            <comment id="80237" author="auto" created="Tue, 17 Jan 2012 06:04:28 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;login&apos;: u&apos;astaple&apos;, u&apos;name&apos;: u&apos;Aaron&apos;, u&apos;email&apos;: u&apos;aaron@10gen.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-4665&quot; title=&quot;Querying against sparse index creates bad state, affecting later queries&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-4665&quot;&gt;&lt;del&gt;SERVER-4665&lt;/del&gt;&lt;/a&gt; backport &apos;ConstraintPresent&apos; constraint type and exclusion of unhelpful recorded plans&lt;br/&gt;
Branch: v2.0&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/0b38e1009b7e43b84fd791cfc58bf83fb3c6d0eb&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/0b38e1009b7e43b84fd791cfc58bf83fb3c6d0eb&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="80181" author="auto" created="Mon, 16 Jan 2012 20:36:45 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;login&apos;: u&apos;astaple&apos;, u&apos;name&apos;: u&apos;Aaron&apos;, u&apos;email&apos;: u&apos;aaron@10gen.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-4665&quot; title=&quot;Querying against sparse index creates bad state, affecting later queries&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-4665&quot;&gt;&lt;del&gt;SERVER-4665&lt;/del&gt;&lt;/a&gt; avoid using an unhelpful cached plan&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/4992d8a270d44301c74fb2facbf1468406a77fa4&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/4992d8a270d44301c74fb2facbf1468406a77fa4&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="80180" author="auto" created="Mon, 16 Jan 2012 20:36:44 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;login&apos;: u&apos;astaple&apos;, u&apos;name&apos;: u&apos;Aaron&apos;, u&apos;email&apos;: u&apos;aaron@10gen.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-4665&quot; title=&quot;Querying against sparse index creates bad state, affecting later queries&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-4665&quot;&gt;&lt;del&gt;SERVER-4665&lt;/del&gt;&lt;/a&gt; simplify QueryPattern model by separating Empty from UpperAndLowerBound constraint types&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/870fd44d145621ed5d6c05bd0311c2b277a9ee37&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/870fd44d145621ed5d6c05bd0311c2b277a9ee37&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="80179" author="auto" created="Mon, 16 Jan 2012 20:36:43 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;login&apos;: u&apos;astaple&apos;, u&apos;name&apos;: u&apos;Aaron&apos;, u&apos;email&apos;: u&apos;aaron@10gen.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-4665&quot; title=&quot;Querying against sparse index creates bad state, affecting later queries&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-4665&quot;&gt;&lt;del&gt;SERVER-4665&lt;/del&gt;&lt;/a&gt; account for $ne bounds in QueryPattern&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/f08b9980df7667642af651a02a3bffd748b82072&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/f08b9980df7667642af651a02a3bffd748b82072&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="79394" author="aaron" created="Thu, 12 Jan 2012 18:39:39 +0000"  >&lt;p&gt;Hi Paul,&lt;/p&gt;

&lt;p&gt;I&apos;ll try to give a brief overview of what&apos;s happening here.  Mongo uses a sampling based mechanism to determine which indexes seem effective for resolving particular queries.  But we don&apos;t use sampling all the time - we also keep a record of indexes that seem to perform well for particular queries and do query pattern matching to attempt to retry those performant indexes for subsequent queries that are similar, but not identical, to earlier queries.&lt;/p&gt;

&lt;p&gt;In this case the $ne part of your query is being ignored for pattern matching, so the index that worked for your first query is being reused for your second query.  And since that index is sparse, using it can alter the result of the query.&lt;/p&gt;

&lt;p&gt;We should probably include your $ne field in the pattern matching model, but will need to figure out exactly how to do that.  When we decide how to do that, I&apos;ll link to the corresponding ticket.&lt;/p&gt;</comment>
                            <comment id="79289" author="canavese" created="Thu, 12 Jan 2012 06:59:56 +0000"  >&lt;p&gt;I also should note that everything works fine if the index is not sparse.&lt;/p&gt;</comment>
                            <comment id="79288" author="canavese" created="Thu, 12 Jan 2012 06:58:47 +0000"  >&lt;p&gt;The formatting didn&apos;t come out well.  Uploading the commands/commentary as an attachment.&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="14384" name="mongo_bug.txt" size="2267" author="canavese" created="Thu, 12 Jan 2012 06:58:47 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>7.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Thu, 12 Jan 2012 18:39:39 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        12 years, 5 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            12 years, 5 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10021"><![CDATA[OS X]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>aaron</customfieldvalue>
            <customfieldvalue>auto</customfieldvalue>
            <customfieldvalue>canavese</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hroh3r:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrgc5b:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>8877</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10166" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Tests Written</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10154"><![CDATA[Complete]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hskkvj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>