<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:21:02 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-29489] Balancer stats generation calls &apos;listDatabases&apos; on shards without maxTimeMS or timeout</title>
                <link>https://jira.mongodb.org/browse/SERVER-29489</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;The sharding balancer&apos;s statistics generation uses &lt;tt&gt;shardutil::retrieveTotalShardSize&lt;/tt&gt; on each round in order to retrieve the shard&apos;s storage utilization.&lt;/p&gt;

&lt;p&gt;This in turn calls &lt;tt&gt;listDatabases&lt;/tt&gt; against the shard, but does not use maxTimeMS or timeout, which can cause the balancer to get stuck indefinitely.&lt;/p&gt;</description>
                <environment></environment>
        <key id="391014">SERVER-29489</key>
            <summary>Balancer stats generation calls &apos;listDatabases&apos; on shards without maxTimeMS or timeout</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="kaloian.manassiev@mongodb.com">Kaloian Manassiev</assignee>
                                    <reporter username="kaloian.manassiev@mongodb.com">Kaloian Manassiev</reporter>
                        <labels>
                    </labels>
                <created>Wed, 7 Jun 2017 06:40:42 +0000</created>
                <updated>Mon, 30 Oct 2023 23:16:18 +0000</updated>
                            <resolved>Tue, 13 Jun 2017 18:01:13 +0000</resolved>
                                    <version>3.4.4</version>
                    <version>3.5.8</version>
                                    <fixVersion>3.4.6</fixVersion>
                    <fixVersion>3.5.9</fixVersion>
                                    <component>Sharding</component>
                                        <votes>0</votes>
                                    <watches>6</watches>
                                                                                                                <comments>
                            <comment id="1601007" author="xgen-internal-githook" created="Mon, 19 Jun 2017 18:44:37 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;kaloianm&apos;, u&apos;name&apos;: u&apos;Kaloian Manassiev&apos;, u&apos;email&apos;: u&apos;kaloian.manassiev@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-29489&quot; title=&quot;Balancer stats generation calls &amp;#39;listDatabases&amp;#39; on shards without maxTimeMS or timeout&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-29489&quot;&gt;&lt;del&gt;SERVER-29489&lt;/del&gt;&lt;/a&gt; Use &apos;maxTimeMs&apos; when calling listDatabases against shards&lt;/p&gt;

&lt;p&gt;(cherry picked from commit 133f0fd284abdf8a313d47b412f58e2cba60cc80)&lt;br/&gt;
Branch: v3.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/ba0eb93a1ae6d80b99542166919ff9e7074cbdf6&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/ba0eb93a1ae6d80b99542166919ff9e7074cbdf6&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1595817" author="xgen-internal-githook" created="Tue, 13 Jun 2017 17:57:42 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;kaloianm&apos;, u&apos;name&apos;: u&apos;Kaloian Manassiev&apos;, u&apos;email&apos;: u&apos;kaloian.manassiev@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-29489&quot; title=&quot;Balancer stats generation calls &amp;#39;listDatabases&amp;#39; on shards without maxTimeMS or timeout&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-29489&quot;&gt;&lt;del&gt;SERVER-29489&lt;/del&gt;&lt;/a&gt; Use &apos;maxTimeMs&apos; when calling listDatabases against shards&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/133f0fd284abdf8a313d47b412f58e2cba60cc80&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/133f0fd284abdf8a313d47b412f58e2cba60cc80&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1592077" author="milkie" created="Fri, 9 Jun 2017 11:59:53 +0000"  >&lt;p&gt;Sure, we could add that as a secret parameter just for sharding.  I&apos;m not sure we would call it &quot;useCachedStats&quot;, as you are simply looking for the data size of all the collections and indexes, rather than what listDatabases gives you today: the actual on-disk size, which takes into account on-disk compression as well as deleted records.&lt;/p&gt;</comment>
                            <comment id="1592061" author="kaloian.manassiev" created="Fri, 9 Jun 2017 11:22:50 +0000"  >&lt;p&gt;I was looking at the &lt;tt&gt;namesOnly&lt;/tt&gt; option and was thinking that perhaps we can add a &lt;tt&gt;useCachedStats&lt;/tt&gt; option or something like that to listDatabases if it is indeed slow, but I would like to see how slow it is in practice.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=milkie&quot; class=&quot;user-hover&quot; rel=&quot;milkie&quot;&gt;milkie&lt;/a&gt;, is adding a &lt;tt&gt;useCachedStats&lt;/tt&gt; parameter to listDatabases something that&apos;s generally workable?&lt;/p&gt;
</comment>
                            <comment id="1592044" author="dan@10gen.com" created="Fri, 9 Jun 2017 10:56:29 +0000"  >&lt;p&gt;Yes, of course the round trips make no sense.  Retracting!  However, i&apos;m pretty sure storage size retrieval doesn&apos;t use cached data, it calls stat(2) on the backing file. That&apos;s why we added the &quot;namesOnly&quot; version of listDatabases in the first place.  On MMAPv1, the number of calls is bounded because all collection/index data is stored in one set of files.&lt;/p&gt;</comment>
                            <comment id="1592004" author="kaloian.manassiev" created="Fri, 9 Jun 2017 09:19:03 +0000"  >&lt;p&gt;This call is used to get the &lt;b&gt;total&lt;/b&gt; shard storage utilization, not on an individual collection basis. With this approach won&apos;t we have to make individual calls to get the statistics for each collection in every database? Seems to me that with large number of collections this would require a lot of roundtrips and might be worse than listDatabases.&lt;/p&gt;

&lt;p&gt;Now that I looked at the &lt;a href=&quot;https://github.com/mongodb/mongo/blob/6a85a4b845d2d365dc6f8a06c3c02824dc1909dc/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp#L736&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;implementation&lt;/a&gt; of the storage size retrieval for WT it seems to come from cached stats, so it should be pretty efficient already. It&apos;s only the &lt;a href=&quot;https://github.com/mongodb/mongo/blob/6a85a4b845d2d365dc6f8a06c3c02824dc1909dc/src/mongo/db/storage/mmap_v1/record_store_v1_base.cpp#L151&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;MMAP V1&apos;s implementation&lt;/a&gt;, which is proportional to the size and might cause some disk I/O while iterating the extents.&lt;/p&gt;</comment>
                            <comment id="1591811" author="dan@10gen.com" created="Fri, 9 Jun 2017 01:15:11 +0000"  >&lt;p&gt;would it be possible to use the collection and index stats directly to calculate this?  It might be faster than getting the &quot;real&quot; storage size per shard because the number is in memory.&lt;/p&gt;</comment>
                            <comment id="1589559" author="kaloian.manassiev" created="Wed, 7 Jun 2017 10:54:39 +0000"  >&lt;p&gt;Yes, this can happen and the balancer round will be slowed down. We discussed the possibility of caching the storage utilization information and only refresh it periodically and asynchronously, but since so far we haven&apos;t seen cases where this has caused problems we haven&apos;t bothered to optimize it.&lt;/p&gt;

&lt;p&gt;Another possible optimization may come from the project to move the autoSplit logic to the shards, in which case we could use the real-time information used by autoSplit to also do utilization estimates.&lt;/p&gt;</comment>
                            <comment id="1589552" author="dan@10gen.com" created="Wed, 7 Jun 2017 10:39:56 +0000"  >&lt;p&gt;Won&apos;t this call also take a potentially very long time on wt instances that have 1000&apos;s of collections, causing a long pause between rounds if it does this on every shard serially?&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                                        </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="14340"><![CDATA[v3.4]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[500A000000XMRRbIAP]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 7 Jun 2017 10:39:56 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        6 years, 34 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            6 years, 34 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>dan@mongodb.com</customfieldvalue>
            <customfieldvalue>milkie@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>kaloian.manassiev@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|ht8p1b:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|ht0qs7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="1705">Sharding 2017-06-19</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11861" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>User Summary</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="11856"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|ht8b3r:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>