<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:27:32 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-31586] Recreate PeriodicBalancerSettingsRefresher on the ServiceContext</title>
                <link>https://jira.mongodb.org/browse/SERVER-31586</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Originally created by &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-30443&quot; title=&quot;Create PeriodicBalancerSettingsRefresher on the ServiceContext&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-30443&quot;&gt;&lt;del&gt;SERVER-30443&lt;/del&gt;&lt;/a&gt;, then removed by &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-31585&quot; title=&quot;Remove PeriodicBalancerSettingsRefresher that is both unused and incorrectly set up&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-31585&quot;&gt;&lt;del&gt;SERVER-31585&lt;/del&gt;&lt;/a&gt; for issues that the ticket delineates.&lt;/p&gt;</description>
                <environment></environment>
        <key id="446233">SERVER-31586</key>
            <summary>Recreate PeriodicBalancerSettingsRefresher on the ServiceContext</summary>
                <type id="3" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14718&amp;avatarType=issuetype">Task</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="matthew.saltz@mongodb.com">Matthew Saltz</assignee>
                                    <reporter username="dianna.hohensee@mongodb.com">Dianna Hohensee</reporter>
                        <labels>
                    </labels>
                <created>Mon, 16 Oct 2017 18:19:35 +0000</created>
                <updated>Mon, 30 Oct 2023 23:12:45 +0000</updated>
                            <resolved>Wed, 25 Jul 2018 21:05:14 +0000</resolved>
                                                    <fixVersion>4.1.2</fixVersion>
                                    <component>Sharding</component>
                                        <votes>0</votes>
                                    <watches>5</watches>
                                                                                                                <comments>
                            <comment id="1956757" author="xgen-internal-githook" created="Wed, 25 Jul 2018 21:04:02 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Matthew Saltz&apos;, &apos;email&apos;: &apos;matthew.saltz@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-31586&quot; title=&quot;Recreate PeriodicBalancerSettingsRefresher on the ServiceContext&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-31586&quot;&gt;&lt;del&gt;SERVER-31586&lt;/del&gt;&lt;/a&gt; Use PeriodicRunner to refresh balancer configuration periodically&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/a80d2ae80e0c55f86284b3a5215bfd396b0d4a49&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/a80d2ae80e0c55f86284b3a5215bfd396b0d4a49&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1942265" author="xgen-internal-githook" created="Mon, 9 Jul 2018 18:46:46 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Matthew Saltz&apos;, &apos;email&apos;: &apos;matthew.saltz@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-31586&quot; title=&quot;Recreate PeriodicBalancerSettingsRefresher on the ServiceContext&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-31586&quot;&gt;&lt;del&gt;SERVER-31586&lt;/del&gt;&lt;/a&gt; Update balancer configuration on mongod on addShard&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/5cbed04d484cc92c3c0b571fd49f16b2ceef81b0&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/5cbed04d484cc92c3c0b571fd49f16b2ceef81b0&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1913670" author="matthew.saltz" created="Thu, 7 Jun 2018 14:36:24 +0000"  >&lt;p&gt;Talked with Randolph online, and we agreed on:&lt;/p&gt;

&lt;ol&gt;
	&lt;li&gt;New command refreshBalancerConfigNow that will be called during addShard on the config server after inserting the shard identity document that triggers sharding initialization&lt;/li&gt;
	&lt;li&gt;Use the onStepUp and onStepDown functions in the ChunkSplitter to trigger start and stop of a periodic refresher, where the first refresh would happen synchronously.&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;This will fix test issues, allow for periodic refreshes, handle step-up and step-down, and handle shutdown and restart. &lt;/p&gt;</comment>
                            <comment id="1913642" author="matthew.saltz" created="Thu, 7 Jun 2018 14:15:58 +0000"  >&lt;p&gt;&#160; &#160; &#160; &#160; &quot;during addShard, also store the balancer config in the new shard&quot;&lt;/p&gt;

&lt;p&gt;So you&apos;d send the balancer configuration inside the document that addShard sends?&lt;/p&gt;

&lt;p&gt;If that&apos;s the suggestion, that would actually fix test issues, and after that I think it would be sufficient to use the onStepUp and onStepDown functions in the ChunkSplitter to trigger start and stop of a periodic refresher, where the first refresh would happen synchronously.&lt;/p&gt;</comment>
                            <comment id="1913622" author="renctan" created="Thu, 7 Jun 2018 14:08:39 +0000"  >&lt;p&gt;What if we treat the balancer config like other sharding metadata (chunk/coll/db)? We persist the info in storage and update when it when we detect a mismatch. So the idea will be:&lt;/p&gt;

&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;during addShard, also store the balancer config in the new shard&lt;/li&gt;
	&lt;li&gt;during sharding initialization, load the stored balancer config&lt;/li&gt;
	&lt;li&gt;whenever balancer config is refreshed and is different from what we have, update the stored config&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;This has the benefit of working also on server restarts.&lt;/p&gt;</comment>
                            <comment id="1913581" author="matthew.saltz" created="Thu, 7 Jun 2018 13:39:59 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=esha.maharishi&quot; class=&quot;user-hover&quot; rel=&quot;esha.maharishi&quot;&gt;esha.maharishi&lt;/a&gt;&#160;had suggested potentially doing the refresh on shard version mismatch, but I think that wouldn&apos;t work until a query for a sharded collection came in. However, we could consider throwing a StaleConfigException (or some new error ShardInitalizationInProgress) on inserts while some flag is set that indicates whether the shard is finished &apos;initializing&apos; after addShard. Then we&apos;d catch the error and then do the refresh. This seems like it could be useful in general potentially - to have some place not under a lock to finish shard initialization that&apos;s allowed to make blocking calls - but it might be a bit extreme to do just for this small change we&apos;re making.&lt;/p&gt;</comment>
                            <comment id="1913177" author="matthew.saltz" created="Wed, 6 Jun 2018 23:19:13 +0000"  >&lt;p&gt;There are two reasons something like this ticket needs to be done:&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;Our tests rely on setting the maximum chunk size in the initialization of ShardingTest, and expect that subsequent inserts will split according to that parameter. So, the shard must refresh from the balancer prior to doing any inserts (or change our tests) if we want our tests to be valid.&lt;/li&gt;
	&lt;li&gt;In general, the logic for &quot;shouldSplit&quot; relies on the current balancer configuration to decide when to split, so if a customer wants to set this value to something smaller than the default, the shard would ideally update it relatively quickly so that chunks don&apos;t grow too large before hearing about the update.&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;There are two things that need to be considered:&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;When to do the first refresh from the config server&lt;/li&gt;
	&lt;li&gt;How often to refresh from the config server&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;For the purposes of tests, all we need is for the first refresh to happen when the shard is added to the cluster with the addShard command. My first thought was to piggy-back on the addShard path in the shard, but no blocking/networking calls can be made in &lt;a href=&quot;https://github.com/mongodb/mongo/blob/master/src/mongo/db/s/sharding_state.cpp#L178&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;ShardingState::initializeFromShardIdentity&lt;/a&gt; because it&apos;s under a lock.&lt;/p&gt;

&lt;p&gt;So my second thought was this &lt;a href=&quot;https://mongodbcr.appspot.com/206720001/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;code review&lt;/a&gt;, which uses ChunkSplitter::setReplicaSetMode (which is called in initializeFromShardIdentity) to launch a thread to do the refresh, which will also occur whenever step-up occurs. The problem with this is that it&apos;s asynchronous and can fail tests non-deterministically and also cause undesirable behavior for customers if for whatever reason it stalls while writes are happening.&lt;/p&gt;

&lt;p&gt;This can be addressed by doing the same thing, by essentially having the ChunkSplitter not do any splits until this refresh is done. This would not, however, block inserts, which means it wouldn&apos;t necessarily fix our tests (though it would fix issues where a customer increases the chunk size and would not want splits to keep happening until the configuration is refreshed).&lt;/p&gt;

&lt;p&gt;Another option would be to have a blocking test-only command refreshBalancerConfigNow that would be used after adding shards in ShardingTest, to fix the testing issues, and then do the previous idea for setReplicaSetMode (which is currently only called on addShard but could be made to happen on onStepUp as well, though there&apos;s no reason onStepUp couldn&apos;t just block and do the refresh).&lt;/p&gt;

&lt;p&gt;I think my favorite idea is a test-only command that blocks and does a refresh, to fix tests. Then we&apos;d have ChunkSplitter start a thread on the first call to setReplicaSetMode to do periodic refreshes, and then stop that thread on onStepDown, and restart it if onStepUp is ever called. For the very first time it starts, we&apos;d ignore requests for splits as described above until some flag like &quot;balancerConfigIsInitialized&quot; is set.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=renctan&quot; class=&quot;user-hover&quot; rel=&quot;renctan&quot;&gt;renctan&lt;/a&gt; &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=schwerin&quot; class=&quot;user-hover&quot; rel=&quot;schwerin&quot;&gt;schwerin&lt;/a&gt; Do you have any better ideas?&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                            <outwardlinks description="depends on">
                                        <issuelink>
            <issuekey id="555574">SERVER-35486</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="558773">SERVER-35585</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>7.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>5.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 6 Jun 2018 23:19:13 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        5 years, 29 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[<s><a href='https://jira.mongodb.org/browse/SERVER-35486'>SERVER-35486</a></s>, <s><a href='https://jira.mongodb.org/browse/SERVER-35585'>SERVER-35585</a></s>]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                            <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>PM-33</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            5 years, 29 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>dianna.hohensee@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>matthew.saltz@mongodb.com</customfieldvalue>
            <customfieldvalue>randolph@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hthybj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr8ttb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="2291">Sharding 2018-06-04</customfieldvalue>
    <customfieldvalue id="2292">Sharding 2018-06-18</customfieldvalue>
    <customfieldvalue id="2385">Sharding 2018-07-02</customfieldvalue>
    <customfieldvalue id="2386">Sharding 2018-07-16</customfieldvalue>
    <customfieldvalue id="2387">Sharding 2018-07-30</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hthkfb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>