<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:13:04 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-6904] Inner Collection (w/ &gt; 10000 entries) with Indexed Fields Dramatically hurt performance when adding new fields on the outer document or entries in the inner collection</title>
                <link>https://jira.mongodb.org/browse/SERVER-6904</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;The following has been tested on mongoDB v2.2.0rc0 &amp;amp; mongoDB v2.2.0:&lt;/p&gt;

&lt;ol&gt;
	&lt;li&gt;
	&lt;ol&gt;
		&lt;li&gt;
		&lt;ol&gt;
			&lt;li&gt;The test:&lt;/li&gt;
		&lt;/ol&gt;
		&lt;/li&gt;
	&lt;/ol&gt;
	&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;First initialize the database:&lt;/p&gt;

&lt;p&gt;    &amp;gt; use test&lt;br/&gt;
    &amp;gt; db.dropDatabase()&lt;br/&gt;
    &amp;gt; db.createCollection(&apos;testdocuments&apos;)&lt;br/&gt;
    &amp;gt; db.testdocuments.insert(&lt;/p&gt;
{ _id : ObjectId(&quot;000000000000000000000001&quot;) }
&lt;p&gt;)&lt;/p&gt;

&lt;p&gt;Then populate a document&apos;s inner collection with objects:&lt;/p&gt;

&lt;p&gt;    &amp;gt; for ( var i = 1; i &amp;lt; 10000; i++) { db.testdocuments.update(&lt;/p&gt;
{ _id : ObjectId(&quot;000000000000000000000001&quot;) }
&lt;p&gt;, { $push : { &apos;a&apos; : &lt;/p&gt;
{ uid1 : i, uid2:  i }
&lt;p&gt; } }) }&lt;/p&gt;

&lt;p&gt;Then add a new outer field:&lt;/p&gt;

&lt;p&gt;    &amp;gt; db.testdocuments.update(&lt;/p&gt;
{ _id : ObjectId(&quot;000000000000000000000001&quot;) }
&lt;p&gt;, { $set : &lt;/p&gt;
{ &apos;aNewOuterField&apos; : 1 }
&lt;p&gt; })&lt;/p&gt;

&lt;p&gt;All is well and the update takes normal time. But, if those inner documents are indexed:&lt;/p&gt;

&lt;p&gt;    &amp;gt; db.testdocuments.ensureIndex(&lt;/p&gt;
{ &apos;a.uid1&apos; : 1 }
&lt;p&gt;)&lt;/p&gt;

&lt;p&gt;    mongodb-console-&amp;gt; Thu Aug 30 14:17:31 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4&amp;#93;&lt;/span&gt; build index test.testdocuments &lt;/p&gt;
{ a.uid1: 1.0 }
&lt;p&gt;    mongodb-console-&amp;gt; Thu Aug 30 14:17:34 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4&amp;#93;&lt;/span&gt; build index done.  scanned 1 total records. 2.244 secs&lt;br/&gt;
    mongodb-console-&amp;gt; Thu Aug 30 14:17:34 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4&amp;#93;&lt;/span&gt; insert test.system.indexes keyUpdates:0 locks(micros) w:2244536 2244ms&lt;/p&gt;

&lt;p&gt;and we add another outer field : &lt;/p&gt;

&lt;p&gt;    &amp;gt; db.testdocuments.update(&lt;/p&gt;
{ _id : ObjectId(&quot;000000000000000000000001&quot;) }
&lt;p&gt;, { $set : &lt;/p&gt;
{ &apos;anotherOuterField&apos; : 1 }
&lt;p&gt; })&lt;/p&gt;

&lt;p&gt;Then we get the MongoDB console reporting the action as slow:&lt;/p&gt;

&lt;p&gt;    mongodb-console-&amp;gt; Thu Aug 30 14:20:01 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4&amp;#93;&lt;/span&gt; update test.testdocuments query: &lt;/p&gt;
{ _id: ObjectId(&apos;000000000000000000000001&apos;) }
&lt;p&gt; update: { $set: &lt;/p&gt;
{ anotherOuterField: 1.0 }
&lt;p&gt; } idhack:1 nupdated:1 keyUpdates:0 locks(micros) w:4191036 4191ms&lt;/p&gt;

&lt;p&gt;Indeed, the last operation has been reported to have taken more than 4 seconds even though there was no index modification at all.&lt;/p&gt;

&lt;p&gt;This might be due to the document being moved but setting the profiler &apos;on&apos;,&lt;/p&gt;

&lt;p&gt;   &amp;gt; db.setProfilingLevel(1)&lt;/p&gt;

&lt;p&gt;and adding another outer field:&lt;/p&gt;

&lt;p&gt;   &amp;gt; db.testdocuments.update(&lt;/p&gt;
{ _id : ObjectId(&quot;000000000000000000000001&quot;) }
&lt;p&gt;, { $set : &lt;/p&gt;
{ &apos;oneLastOuterField&apos; : 1 }
&lt;p&gt; })&lt;/p&gt;

&lt;p&gt;gives us the following:&lt;/p&gt;

&lt;p&gt;   &amp;gt; db.system.profile.find().pretty()&lt;/p&gt;

&lt;p&gt;      {&lt;br/&gt;
        &quot;ts&quot; : ISODate(&quot;2012-08-30T18:23:03.053Z&quot;),&lt;br/&gt;
        &quot;op&quot; : &quot;update&quot;,&lt;br/&gt;
        &quot;ns&quot; : &quot;test.testdocuments&quot;,&lt;br/&gt;
        &quot;query&quot; : &lt;/p&gt;
{
          &quot;_id&quot; : ObjectId(&quot;000000000000000000000001&quot;)
        }
&lt;p&gt;,&lt;br/&gt;
        &quot;updateobj&quot; : {&lt;br/&gt;
          &quot;$set&quot; : &lt;/p&gt;
{
            &quot;oneLastOuterField&quot; : 1
          }
&lt;p&gt;        },&lt;br/&gt;
        &quot;idhack&quot; : true,&lt;br/&gt;
        &quot;nupdated&quot; : 1,&lt;br/&gt;
        &quot;keyUpdates&quot; : 0,&lt;br/&gt;
        &quot;numYield&quot; : 0,&lt;br/&gt;
        &quot;lockStats&quot; : {&lt;br/&gt;
          &quot;timeLockedMicros&quot; : &lt;/p&gt;
{
            &quot;r&quot; : NumberLong(0),
            &quot;w&quot; : NumberLong(4064791)
          }
&lt;p&gt;,&lt;br/&gt;
          &quot;timeAcquiringMicros&quot; : &lt;/p&gt;
{
            &quot;r&quot; : NumberLong(0),
            &quot;w&quot; : NumberLong(47)
          }
&lt;p&gt;        },&lt;br/&gt;
        &quot;millis&quot; : 4064,&lt;br/&gt;
        &quot;client&quot; : &quot;127.0.0.1&quot;,&lt;br/&gt;
        &quot;user&quot; : &quot;&quot;&lt;br/&gt;
      }&lt;/p&gt;

&lt;p&gt;which definitely shows that the delay isn&apos;t due to the document being &apos;moved&apos;.&lt;/p&gt;

&lt;p&gt;Adding to the inner collection is not faster either as &lt;/p&gt;

&lt;p&gt;    &amp;gt; db.testdocuments.update(&lt;/p&gt;
{ _id : ObjectId(&quot;000000000000000000000001&quot;) }
&lt;p&gt;, { $push : { &apos;a&apos; : &lt;/p&gt;
{ uid1 : -1, uid2: -1 }
&lt;p&gt; } })&lt;/p&gt;

&lt;p&gt;gives us the following profile&lt;/p&gt;

&lt;p&gt;    &amp;gt; db.system.profile.find().pretty() &lt;/p&gt;

&lt;p&gt;      {&lt;br/&gt;
        &quot;ts&quot; : ISODate(&quot;2012-08-30T18:32:34.080Z&quot;),&lt;br/&gt;
        &quot;op&quot; : &quot;update&quot;,&lt;br/&gt;
        &quot;ns&quot; : &quot;test.testdocuments&quot;,&lt;br/&gt;
        &quot;query&quot; : &lt;/p&gt;
{
          &quot;_id&quot; : ObjectId(&quot;000000000000000000000001&quot;)
        }
&lt;p&gt;,&lt;br/&gt;
        &quot;updateobj&quot; : {&lt;br/&gt;
          &quot;$push&quot; : {&lt;br/&gt;
            &quot;a&quot; : &lt;/p&gt;
{
              &quot;uid1&quot; : -1,
              &quot;uid2&quot; : -1
            }
&lt;p&gt;          }&lt;br/&gt;
        },&lt;br/&gt;
        &quot;nscanned&quot; : 1,&lt;br/&gt;
        &quot;nupdated&quot; : 1,&lt;br/&gt;
        &quot;keyUpdates&quot; : 1,&lt;br/&gt;
        &quot;numYield&quot; : 0,&lt;br/&gt;
        &quot;lockStats&quot; : {&lt;br/&gt;
          &quot;timeLockedMicros&quot; : &lt;/p&gt;
{
            &quot;r&quot; : NumberLong(0),
            &quot;w&quot; : NumberLong(4057496)
          }
&lt;p&gt;,&lt;br/&gt;
          &quot;timeAcquiringMicros&quot; : &lt;/p&gt;
{
            &quot;r&quot; : NumberLong(0),
            &quot;w&quot; : NumberLong(213)
          }
&lt;p&gt;        },&lt;br/&gt;
        &quot;millis&quot; : 4057,&lt;br/&gt;
        &quot;client&quot; : &quot;127.0.0.1&quot;,&lt;br/&gt;
        &quot;user&quot; : &quot;&quot;&lt;br/&gt;
      }&lt;/p&gt;</description>
                <environment>Behavior witnessed on OS X &amp;amp; Ubuntu 12.04</environment>
        <key id="48675">SERVER-6904</key>
            <summary>Inner Collection (w/ &gt; 10000 entries) with Indexed Fields Dramatically hurt performance when adding new fields on the outer document or entries in the inner collection</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="3">Duplicate</resolution>
                                        <assignee username="aaron">Aaron Staple</assignee>
                                    <reporter username="glesperance">Gabriel Lesperance</reporter>
                        <labels>
                    </labels>
                <created>Thu, 30 Aug 2012 18:57:31 +0000</created>
                <updated>Fri, 15 Feb 2013 15:06:27 +0000</updated>
                            <resolved>Mon, 21 Jan 2013 20:41:11 +0000</resolved>
                                    <version>2.2.0-rc0</version>
                    <version>2.2.0</version>
                                                                        <votes>0</votes>
                                    <watches>4</watches>
                                                                                                                <comments>
                            <comment id="245864" author="glesperance" created="Mon, 21 Jan 2013 20:45:48 +0000"  >&lt;p&gt;No problem. Thanks for the support.&lt;/p&gt;</comment>
                            <comment id="245859" author="aaron" created="Mon, 21 Jan 2013 20:41:11 +0000"  >&lt;p&gt;Hi Gabriel - I&apos;m going to resolve this as a duplicate of &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-8192&quot; title=&quot;Optimize btree key generation&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-8192&quot;&gt;&lt;del&gt;SERVER-8192&lt;/del&gt;&lt;/a&gt; and &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-8193&quot; title=&quot;Optimize in place updates that modify an index (not to deteriorate based on size of a different indexed field)&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-8193&quot;&gt;&lt;del&gt;SERVER-8193&lt;/del&gt;&lt;/a&gt;.  Please feel free to reopen if you have additional questions.&lt;/p&gt;</comment>
                            <comment id="241690" author="aaron" created="Wed, 16 Jan 2013 18:44:38 +0000"  >&lt;p&gt;Hi Gabriel,&lt;/p&gt;

&lt;p&gt;When an update is requested that can&apos;t be accomplished by simply modifying the value of an existing field in place (think $inc modifying a field in place), and the document does not need to move to a new disk location, the update code will do the following:&lt;/p&gt;

&lt;p&gt;1) extract all index keys for the old version of the document&lt;br/&gt;
2) extract all index keys for the new version of the document&lt;br/&gt;
3) find the index keys that were added or removed, and update the indexes appropriately&lt;/p&gt;

&lt;p&gt;The slowdown you are seeing results from steps 1 and 2.  After you add your a.uid index, steps 1 and 2 each need to extract 10k keys and store them in an std::set.  (Before you added the index in your test, steps 1 and 2 did not need to be run for this index.)&lt;/p&gt;

&lt;p&gt;I&apos;ve filed two tickets to work on relevant optimizations:&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-8192&quot; title=&quot;Optimize btree key generation&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-8192&quot;&gt;&lt;del&gt;SERVER-8192&lt;/del&gt;&lt;/a&gt; is for optimizing the code that extracts the index keys from a document&lt;br/&gt;
&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-8193&quot; title=&quot;Optimize in place updates that modify an index (not to deteriorate based on size of a different indexed field)&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-8193&quot;&gt;&lt;del&gt;SERVER-8193&lt;/del&gt;&lt;/a&gt; is for optimizing update&apos;s index handling code in cases where there are no mods on fields belonging to the index&lt;/p&gt;</comment>
                            <comment id="235169" author="glesperance" created="Wed, 9 Jan 2013 15:49:41 +0000"  >&lt;p&gt;Any progress on this?&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                            <outwardlinks description="duplicates">
                                        <issuelink>
            <issuekey id="62184">SERVER-8192</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="62185">SERVER-8193</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>4.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 16 Jan 2013 18:44:38 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        11 years, 4 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ian@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            11 years, 4 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10000" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Old_Backport</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10000"><![CDATA[No]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>aaron</customfieldvalue>
            <customfieldvalue>glesperance</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrnqa7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrg413:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>7557</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|ht06hr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>