<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:21:53 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-29817] Optimize incremental update performance of ChunkManager and CollectionMetadata</title>
                <link>https://jira.mongodb.org/browse/SERVER-29817</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;The &lt;a href=&quot;https://github.com/mongodb/mongo/blob/master/src/mongo/s/chunk_manager.h#L56&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;ChunkManager&lt;/a&gt; and &lt;a href=&quot;https://github.com/mongodb/mongo/blob/master/src/mongo/db/s/collection_metadata.h#L53&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;CollectionMetadata&lt;/a&gt; types are effectively in-memory routing tables used by map shard key values to chunks and shards. These data structures are read only, and updated copies must be made after chunk migrations. Because the data structures are updated by copy, their update time is proportional to the total number of chunks in a given collection. When the number of chunks in a collection is large (10s of thousands), the update time is significant, and increases the latency of chunk migration commit.&lt;/p&gt;

&lt;p&gt;This ticket tracks improvements to the constant factors affecting update time. Prototyping indicates that a speedup of &amp;gt;4x is possible by replacing woCompare with KeyString comparison, and by replacing all but one instance of std::map with instances of std::vector that are constructed in sorted order.&lt;/p&gt;</description>
                <environment></environment>
        <key id="397478">SERVER-29817</key>
            <summary>Optimize incremental update performance of ChunkManager and CollectionMetadata</summary>
                <type id="4" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14710&amp;avatarType=issuetype">Improvement</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="schwerin@mongodb.com">Andy Schwerin</assignee>
                                    <reporter username="schwerin@mongodb.com">Andy Schwerin</reporter>
                        <labels>
                    </labels>
                <created>Fri, 23 Jun 2017 14:24:04 +0000</created>
                <updated>Mon, 30 Oct 2023 23:15:46 +0000</updated>
                            <resolved>Fri, 21 Jul 2017 12:53:48 +0000</resolved>
                                                    <fixVersion>3.4.7</fixVersion>
                    <fixVersion>3.5.11</fixVersion>
                                    <component>Sharding</component>
                                        <votes>3</votes>
                                    <watches>21</watches>
                                                                                                                <comments>
                            <comment id="1632294" author="xgen-internal-githook" created="Wed, 26 Jul 2017 15:26:23 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;email&apos;: &apos;schwerin@mongodb.com&apos;, &apos;username&apos;: &apos;andy10gen&apos;, &apos;name&apos;: &apos;Andy Schwerin&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-29817&quot; title=&quot;Optimize incremental update performance of ChunkManager and CollectionMetadata&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-29817&quot;&gt;&lt;del&gt;SERVER-29817&lt;/del&gt;&lt;/a&gt; Use hinted insert when building CollectionMetadata for performance.&lt;br/&gt;
Branch: v3.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/84a658596bc62107539a07556b9c066af2fec683&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/84a658596bc62107539a07556b9c066af2fec683&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1632293" author="xgen-internal-githook" created="Wed, 26 Jul 2017 15:26:22 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;email&apos;: &apos;schwerin@mongodb.com&apos;, &apos;username&apos;: &apos;andy10gen&apos;, &apos;name&apos;: &apos;Andy Schwerin&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-29817&quot; title=&quot;Optimize incremental update performance of ChunkManager and CollectionMetadata&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-29817&quot;&gt;&lt;del&gt;SERVER-29817&lt;/del&gt;&lt;/a&gt; Use hinted insert when building chunkRangeMap&lt;br/&gt;
Branch: v3.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/01a38984d5c7b134b177d1b78023ed779c8b1631&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/01a38984d5c7b134b177d1b78023ed779c8b1631&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1628429" author="schwerin" created="Fri, 21 Jul 2017 12:53:48 +0000"  >&lt;p&gt;I&apos;m going to wrap this ticket up with these minor optimizations, which are suitable for backport and provide substantial speed up to incremental routing table refresh in systems with thousands of chunks.&lt;/p&gt;

&lt;p&gt;The following summarizes the time to refresh the routing data structures in a system with 50,000 chunks. &quot;rebuild&quot; is for a full build of the tables from scratch, and &quot;move 1&quot; is for updating the structures after moving 1 chunk.&lt;/p&gt;

&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;&amp;nbsp;&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;rebuild &lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt; move 1 &lt;/th&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;without optimization     &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 451ms   &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 189ms &lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;this ticket &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 312ms   &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;  47ms &lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;


&lt;p&gt;Most of the structures are completely rebuilt after even incremental updates, and that process takes time proportional to the number of chunks. You can see the cost of those changes in the &quot;move 1&quot; column. The changes in this ticket transform those rebuilds from O(n log n) to O&amp;#40;n) by leveraging the fact that we construct those structures in sorted-order.&lt;/p&gt;

&lt;p&gt;Further optimization is possible, but they are more invasive than these changes.&lt;/p&gt;</comment>
                            <comment id="1628422" author="xgen-internal-githook" created="Fri, 21 Jul 2017 12:47:36 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;andy10gen&apos;, u&apos;name&apos;: u&apos;Andy Schwerin&apos;, u&apos;email&apos;: u&apos;schwerin@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-29817&quot; title=&quot;Optimize incremental update performance of ChunkManager and CollectionMetadata&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-29817&quot;&gt;&lt;del&gt;SERVER-29817&lt;/del&gt;&lt;/a&gt; Use hinted insert when building chunkRangeMap&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/43441676092ae87f4d0cc1bf81877f9610149454&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/43441676092ae87f4d0cc1bf81877f9610149454&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1628421" author="xgen-internal-githook" created="Fri, 21 Jul 2017 12:47:35 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;andy10gen&apos;, u&apos;name&apos;: u&apos;Andy Schwerin&apos;, u&apos;email&apos;: u&apos;schwerin@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-29817&quot; title=&quot;Optimize incremental update performance of ChunkManager and CollectionMetadata&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-29817&quot;&gt;&lt;del&gt;SERVER-29817&lt;/del&gt;&lt;/a&gt; Use hinted insert when building CollectionMetadata for performance.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/a045e38ca392f8354ea85ec5cebfb6d52892f444&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/a045e38ca392f8354ea85ec5cebfb6d52892f444&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1620365" author="xgen-internal-githook" created="Wed, 12 Jul 2017 19:30:35 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;andy10gen&apos;, u&apos;name&apos;: u&apos;Andy Schwerin&apos;, u&apos;email&apos;: u&apos;schwerin@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-29817&quot; title=&quot;Optimize incremental update performance of ChunkManager and CollectionMetadata&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-29817&quot;&gt;&lt;del&gt;SERVER-29817&lt;/del&gt;&lt;/a&gt; Move construction of CollectionMetadata data structures out of ShardingState.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/4b9d69eb00361083ce835d42c4107a4caa52f6fc&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/4b9d69eb00361083ce835d42c4107a4caa52f6fc&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1620364" author="xgen-internal-githook" created="Wed, 12 Jul 2017 19:30:34 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;andy10gen&apos;, u&apos;name&apos;: u&apos;Andy Schwerin&apos;, u&apos;email&apos;: u&apos;schwerin@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-29817&quot; title=&quot;Optimize incremental update performance of ChunkManager and CollectionMetadata&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-29817&quot;&gt;&lt;del&gt;SERVER-29817&lt;/del&gt;&lt;/a&gt; Move all ChunkManager construction logic into chunk_manager.cpp; hide implementation details.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/4c15828d7bd7222fbcb5dc5b3c2060ea2c136dc7&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/4c15828d7bd7222fbcb5dc5b3c2060ea2c136dc7&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1614498" author="xgen-internal-githook" created="Wed, 5 Jul 2017 19:28:21 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;andy10gen&apos;, u&apos;name&apos;: u&apos;Andy Schwerin&apos;, u&apos;email&apos;: u&apos;schwerin@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-29817&quot; title=&quot;Optimize incremental update performance of ChunkManager and CollectionMetadata&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-29817&quot;&gt;&lt;del&gt;SERVER-29817&lt;/del&gt;&lt;/a&gt; Do not use chunk map data structure in ChunkManager interface.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/74ad9fa5538c82478aced69a1969266752b1d7a8&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/74ad9fa5538c82478aced69a1969266752b1d7a8&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1614497" author="xgen-internal-githook" created="Wed, 5 Jul 2017 19:28:20 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;andy10gen&apos;, u&apos;name&apos;: u&apos;Andy Schwerin&apos;, u&apos;email&apos;: u&apos;schwerin@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-29817&quot; title=&quot;Optimize incremental update performance of ChunkManager and CollectionMetadata&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-29817&quot;&gt;&lt;del&gt;SERVER-29817&lt;/del&gt;&lt;/a&gt; Always intialize bytesWritten to 0 for chunks.&lt;/p&gt;

&lt;p&gt;Previously, it was initialized to a pseudorandom value that was less than&lt;br/&gt;
some fraction of the maximum chunk size. This introduced a depencency between&lt;br/&gt;
the chunk constuctor and the balancer, and provided no particular value.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/9d803b41497eb54864361a3877097d8e1dd55dc2&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/9d803b41497eb54864361a3877097d8e1dd55dc2&lt;/a&gt;&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                                                <inwardlinks description="is duplicated by">
                                        <issuelink>
            <issuekey id="215105">SERVER-19295</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                                                <inwardlinks description="is related to">
                                                        </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="14340"><![CDATA[v3.4]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[500A000000YQAWhIAP, 500A000000UaQVYIA3]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 5 Jul 2017 19:28:20 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        6 years, 29 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            6 years, 29 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>schwerin@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|ht9sov:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|ht1rp3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|ht9erb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>