<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 06:34:24 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-77057] Investigate slow down caused by OpObservers in updateMany code path</title>
                <link>https://jira.mongodb.org/browse/SERVER-77057</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;While investigating&#160;&lt;a href=&quot;https://jira.mongodb.org/browse/PERF-3818&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;https://jira.mongodb.org/browse/PERF-3818&lt;/a&gt; have identified a regression under mongo::OpObserverRegistry::onUpdate (mongo::OpObserverRegistry::onUpdate: 10.17% in 7.0 but 5.11% in 6.0 while 6.0 containing fewer samples). Partly it&apos;s due to adding a new observer (QueryAnalysisOpObserver), partly because more work done in ClusterServerParameterOpObserver and OpObserverImpl, and partly due to an increased overhead across all observers.&lt;/p&gt;

&lt;p&gt;Should look into:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;what is the shared overhead? (could it be due to replacing cached nss with virtual coll-&amp;gt;ns() call?)&lt;/li&gt;
	&lt;li&gt;could any of the observers be trimmed or skipped in a more efficient way for updateMany?&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;I&apos;ve added local std::chrono-based logging in the loop of OpObserverRegistry::onUpdate and saw the following:&lt;/p&gt;
&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;v6.0&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Observer&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;v7.0&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Observer&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Comment&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;100.64&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;AuthOpObserver&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;173.72&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;AuthOpObserver&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Calls &apos;AuthorizationManagerImpl::logOp&apos; and &apos;audit::logUpdateOperation&apos;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;53.1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;ClusterServerParameterOpObserver&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;105.66&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;ClusterServerParameterOpObserver&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Moved updateParameter to recoveryUnit&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;53.86&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;FcvOpObserver&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;64.9&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;FcvOpObserver&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;No changes between releases. Does little.&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;447.16&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;OpObserverImpl&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;618.46&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;OpObserverImpl&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Multiple changes&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;55.2&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;PrimaryOnlyServiceOpObserver&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;47.24&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;PrimaryOnlyServiceOpObserver&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Does nothing on update in both releases&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;56.4&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;ShardSplitDonorOpObserver&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;111.78&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;QueryAnalysisOpObserver&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;New in 7.0. The design seems wrong for updateMulti&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;59.38&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;TenantMigrationDonorOpObserver&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;70.24&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;TenantMigrationDonorOpObserver&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Checks whether nss is the one it cares about and does nothing if not&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;58.66&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;TenantMigrationRecipientOpObserver&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;56.66&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;TenantMigrationRecipientOpObserver&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Checks whether nss is the one it cares about and does nothing if not&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;93.78&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;UserWriteBlockModeOpObserver&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;79.92&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;UserWriteBlockModeOpObserver&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;Calls &apos;_checkWriteAllowed()&apos; then checks whether nss is the one it cases about and does nothing if not&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;978.18&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&#160;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1328.58&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&#160;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&#160;&lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;


&lt;p&gt;The numbers are nanoseconds per updated doc, averaged across updateMany that touched 50K docs. The last row is the sum across all observers. In my local tests the sum for 6.0 fluctuated between 900 and 1000. Notice, that even the observers that haven&apos;t changed and do very little in the scenario of this benchmark (essentially, a guard check or two before bailing out), such as TenantMigrationDonorOpObserver still are generally a little slower in 7.0. More contention on the instruction cache, etc.?&#160;&lt;/p&gt;</description>
                <environment></environment>
        <key id="2338676">SERVER-77057</key>
            <summary>Investigate slow down caused by OpObservers in updateMany code path</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="kshitij.gupta@mongodb.com">Kshitij Gupta</assignee>
                                    <reporter username="irina.yatsenko@mongodb.com">Irina Yatsenko</reporter>
                        <labels>
                    </labels>
                <created>Thu, 11 May 2023 21:56:01 +0000</created>
                <updated>Thu, 27 Jul 2023 20:29:02 +0000</updated>
                            <resolved>Thu, 27 Jul 2023 20:29:02 +0000</resolved>
                                                                                        <votes>0</votes>
                                    <watches>18</watches>
                                                                                                                <comments>
                            <comment id="5471717" author="JIRAUSER1253399" created="Fri, 2 Jun 2023 20:24:45 +0000"  >&lt;p&gt;I used google benchmark to evaluate the performance of QueryAnalysisOpObserver, and my changes cut down the time spent in this OpObserver by ~50%.&lt;/p&gt;

&lt;p&gt;&#160;&lt;br/&gt;
------------------------------------------------------------------------&lt;br/&gt;
Benchmark Time CPU Iterations&lt;br/&gt;
------------------------------------------------------------------------&lt;br/&gt;
With changes:&lt;br/&gt;
BM_QueryAnalysisOpObserver/1 14.5 ns 14.5 ns 48061568&lt;br/&gt;
Without changes:&lt;br/&gt;
BM_QueryAnalysisOpObserver/1 28.5 ns 28.5 ns 24595548&lt;br/&gt;
&#160;&lt;br/&gt;
&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-77758&quot; title=&quot;Break down query analysis op observer depending on cluster role.&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-77758&quot;&gt;&lt;del&gt;SERVER-77758&lt;/del&gt;&lt;/a&gt; will be tracking this work.&lt;/p&gt;</comment>
                            <comment id="5468911" author="JIRAUSER1263310" created="Thu, 1 Jun 2023 21:09:40 +0000"  >&lt;p&gt;The prior code review for &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-77057&quot; title=&quot;Investigate slow down caused by OpObservers in updateMany code path&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-77057&quot;&gt;&lt;del&gt;SERVER-77057&lt;/del&gt;&lt;/a&gt; should stack with &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-77364&quot; title=&quot;Speed up OpObservers with a filter framework in OpObserverRegistry&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-77364&quot;&gt;&lt;del&gt;SERVER-77364&lt;/del&gt;&lt;/a&gt;. I hesitate to dive too deep on further investigation before these tickets can both be measured against the regression because they should yield the easiest improvements.&lt;/p&gt;</comment>
                            <comment id="5468906" author="JIRAUSER1263310" created="Thu, 1 Jun 2023 21:06:42 +0000"  >&lt;p&gt;I am linking this ticket to &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-77364&quot; title=&quot;Speed up OpObservers with a filter framework in OpObserverRegistry&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-77364&quot;&gt;&lt;del&gt;SERVER-77364&lt;/del&gt;&lt;/a&gt; which should provide some speedup in one of the hottest functions in the linked ticket: &lt;tt&gt;OpObserverRegistry::onUpdate&lt;/tt&gt;. It should also impact &lt;tt&gt;onInserts&lt;/tt&gt; and potentially &lt;tt&gt;onDelete&lt;/tt&gt;. The PR is in review and I am profiling the change at the moment to see what improvement in can provide.&lt;/p&gt;</comment>
                            <comment id="5426570" author="JIRAUSER1259274" created="Mon, 15 May 2023 21:31:07 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=judah.schvimer%40mongodb.com&quot; class=&quot;user-hover&quot; rel=&quot;judah.schvimer@mongodb.com&quot;&gt;judah.schvimer@mongodb.com&lt;/a&gt;, there was a lot of churn in op_observer_impl.cpp between 6.0 and 7.0. At the function level, ideally, we should have micro-benchmarks to see the impact of individual changes, from a code review I would suspect a few that might have contributed to the longer running time (but keep in mind that stuff gets inlined, etc.):&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;args.nss became rgs.coll-&amp;gt;ns() - the latter is a virtual call and it&apos;s done &lt;em&gt;multiple&lt;/em&gt; times&lt;/li&gt;
	&lt;li&gt;the addition of inBatchedWrite block (this benchmark executes the final &quot;else&quot; clause but it has to compute the flag)&lt;/li&gt;
	&lt;li&gt;&#160;it looks like replLogUpdate became slightly heavier&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;I tried caching nss on OplogUpdateEntryArgs to remove the virtual calls. It saved ~100 nanoseconds per call (about half of the regression). I have no idea whether caching like this would be correct.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                            <outwardlinks description="depends on">
                                        <issuelink>
            <issuekey id="2357341">SERVER-77758</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="2347007">SERVER-77364</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="2347276">SERVER-77373</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="2347007">SERVER-77364</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>4.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_12751" key="com.atlassian.jira.plugin.system.customfieldtypes:multiselect">
                        <customfieldname>Assigned Teams</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="25134"><![CDATA[Sharding NYC]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 12 May 2023 17:15:29 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        35 weeks, 5 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[<s><a href='https://jira.mongodb.org/browse/SERVER-77364'>SERVER-77364</a></s>, <s><a href='https://jira.mongodb.org/browse/SERVER-77758'>SERVER-77758</a></s>]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>kshitij.gupta@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            35 weeks, 5 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>irina.yatsenko@mongodb.com</customfieldvalue>
            <customfieldvalue>kshitij.gupta@mongodb.com</customfieldvalue>
            <customfieldvalue>matt.kneiser@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i28tlz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|i1r3lk:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i28frb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>