<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:09:09 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-45565] Ensure ops inside of a &gt;16 MB transaction with a command do not get batched with ops outside of the transaction</title>
                <link>https://jira.mongodb.org/browse/SERVER-45565</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;This ticket came from a conversation about &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-44852&quot; title=&quot;Have secondary oplog application handle transactions with commands&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-44852&quot;&gt;&lt;del&gt;SERVER-44852&lt;/del&gt;&lt;/a&gt;. As of the changes in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-44852&quot; title=&quot;Have secondary oplog application handle transactions with commands&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-44852&quot;&gt;&lt;del&gt;SERVER-44852&lt;/del&gt;&lt;/a&gt;, transactions with commands are intended to be processed as part of their own batch, separate from any other operations. However, there exists a bug with that code that allows transactions &amp;gt; 16 MB to be batched with operations outside of the transaction. This ticket should address that bug.&lt;/p&gt;</description>
                <environment></environment>
        <key id="1090137">SERVER-45565</key>
            <summary>Ensure ops inside of a &gt;16 MB transaction with a command do not get batched with ops outside of the transaction</summary>
                <type id="3" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14718&amp;avatarType=issuetype">Task</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="andrew.chen@mongodb.com">Andrew Chen</assignee>
                                    <reporter username="maria.vankeulen@mongodb.com">Maria van Keulen</reporter>
                        <labels>
                    </labels>
                <created>Tue, 14 Jan 2020 18:04:50 +0000</created>
                <updated>Sun, 29 Oct 2023 22:13:17 +0000</updated>
                            <resolved>Wed, 4 Mar 2020 23:33:01 +0000</resolved>
                                                    <fixVersion>4.4.0-rc0</fixVersion>
                                    <component>Replication</component>
                                        <votes>0</votes>
                                    <watches>9</watches>
                                                                                                                <comments>
                            <comment id="2948369" author="xgen-internal-githook" created="Fri, 6 Mar 2020 15:29:34 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Andrew Chen&apos;, &apos;email&apos;: &apos;andrew.chen@10gen.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-45565&quot; title=&quot;Ensure ops inside of a &amp;gt;16 MB transaction with a command do not get batched with ops outside of the transaction&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-45565&quot;&gt;&lt;del&gt;SERVER-45565&lt;/del&gt;&lt;/a&gt;: Modified Oplog Batcher behavior to process large transaction oplog entries in separate batches&lt;/p&gt;

&lt;p&gt; create mode 100644 jstests/noPassthrough/large_txn_correctness.js&lt;/p&gt;

&lt;p&gt;(cherry picked from commit 9557a35b779a5f6c1a2453514ec9cc67119b288b)&lt;/p&gt;

&lt;p&gt; create mode 100644 jstests/noPassthrough/large_txn_correctness.js&lt;br/&gt;
Branch: v4.4&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/9dbf423b7705e672ad396d4223e0bfd75a6156f3&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/9dbf423b7705e672ad396d4223e0bfd75a6156f3&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2935434" author="xgen-internal-githook" created="Wed, 4 Mar 2020 22:25:48 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;name&apos;: &apos;Andrew Chen&apos;, &apos;email&apos;: &apos;andrew.chen@10gen.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-45565&quot; title=&quot;Ensure ops inside of a &amp;gt;16 MB transaction with a command do not get batched with ops outside of the transaction&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-45565&quot;&gt;&lt;del&gt;SERVER-45565&lt;/del&gt;&lt;/a&gt;: Modified Oplog Batcher behavior to process large transaction oplog entries in separate batches&lt;/p&gt;

&lt;p&gt; create mode 100644 jstests/noPassthrough/large_txn_correctness.js&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/9557a35b779a5f6c1a2453514ec9cc67119b288b&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/9557a35b779a5f6c1a2453514ec9cc67119b288b&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2907450" author="schwerin" created="Tue, 25 Feb 2020 15:53:47 +0000"  >&lt;p&gt;Thanks,&#160;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=matthew.russotto&quot; class=&quot;user-hover&quot; rel=&quot;matthew.russotto&quot;&gt;matthew.russotto&lt;/a&gt;. &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=geert.bosch&quot; class=&quot;user-hover&quot; rel=&quot;geert.bosch&quot;&gt;geert.bosch&lt;/a&gt;, I&apos;m convinced.&lt;/p&gt;</comment>
                            <comment id="2907241" author="matthew.russotto" created="Tue, 25 Feb 2020 14:38:19 +0000"  >&lt;p&gt;I think &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=geert.bosch&quot; class=&quot;user-hover&quot; rel=&quot;geert.bosch&quot;&gt;geert.bosch&lt;/a&gt;&apos;s proposed solution is probably the simplest and is correct at least for createCollection.  If we were to allow dropCollection, we&apos;d need to process the entire batch individually (avoiding batching with oplog entries that are before the transaction as well).  To be consistent with under-16MB transactions with commands I think adding multi-operation transaction commits to mustProcessIndividually is probably better.  &lt;/p&gt;

&lt;p&gt;As for performance, I don&apos;t know how bad the effect of essentially reducing batch size from 100MB to 16MB for strings of just-over-16MB transactions would be.  Might be worth a test.&lt;/p&gt;

&lt;p&gt;I think the problem is the batcher used to be able know the batch constraints, and with multi-document transactions it no longer can.  We could in principle fix this by reading the entire transaction in the batcher, applying batch limits and rules, and passing it to the applier.  This is not difficult in principle but I think gets complicated in the code, and is probably not worth it unless the performance issues are significant.&lt;/p&gt;</comment>
                            <comment id="2905803" author="siyuan.zhou@10gen.com" created="Mon, 24 Feb 2020 20:11:48 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=matthew.russotto&quot; class=&quot;user-hover&quot; rel=&quot;matthew.russotto&quot;&gt;matthew.russotto&lt;/a&gt;, could you please help evaluate the proposal for correctness and performance concerns? Matthew implemented the batching logic of large transactions.&lt;/p&gt;</comment>
                            <comment id="2889099" author="geert.bosch" created="Fri, 21 Feb 2020 18:27:11 +0000"  >&lt;p&gt;If a customer has a workload with many &amp;gt;16MB transactions, there is a good chance that multiple commits happen in a single batch, inflating the batch size beyond intended limits, with likelihood of substandard performance or even failure on smaller systems. In the failure case, restarting may well retry the exact same scenario with more failures as result.&lt;/p&gt;

&lt;p&gt;So, when large transactions are common, we need to avoid overly large batches to avoid thrashing and crashing. Even for a high oplog write rate of 250 MB/sec, we&apos;d only have 15 batches per second, so the effect of fixed per batch processing cost on overall performance should be modest. If &amp;gt; 16MB transactions are rare (the overwhelmingly common case, I&apos;d hope), there is no performance implication whatsoever.&lt;/p&gt;

&lt;p&gt;In the end, I think that stability and limiting memory pressure in the presence of large transactions trumps potential performance impact of batching too much. If we want to have more sophisticated batch logic at some point, that is fine with me, but today doesn&apos;t seem to be the right time.&lt;/p&gt;</comment>
                            <comment id="2887619" author="maria.vankeulen" created="Thu, 20 Feb 2020 22:40:16 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=schwerin&quot; class=&quot;user-hover&quot; rel=&quot;schwerin&quot;&gt;schwerin&lt;/a&gt; We believe this solution to be the least complex way to resolve the bug described in the ticket. Otherwise, it seems like nontrivial, potentially risky, logic needs to be added to the batcher to properly group the operations inside a &amp;gt;16 MB transaction that executes a command.&lt;/p&gt;

&lt;p&gt;CC &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=geert.bosch&quot; class=&quot;user-hover&quot; rel=&quot;geert.bosch&quot;&gt;geert.bosch&lt;/a&gt; &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=suganthi.mani&quot; class=&quot;user-hover&quot; rel=&quot;suganthi.mani&quot;&gt;suganthi.mani&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2874540" author="schwerin" created="Wed, 12 Feb 2020 18:14:39 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=geert.bosch&quot; class=&quot;user-hover&quot; rel=&quot;geert.bosch&quot;&gt;geert.bosch&lt;/a&gt;, while I see how that heuristic might make resolving this bug easier, I don&apos;t love adding more limitations to when a batch boundary must be drawn. I also don&apos;t know that I agree that a single transaction that&apos;s just over 16MB is necessarily a reasonable limit to the&#160; amount of work for a single replication batch. What&apos;s your justification?&lt;/p&gt;

&lt;p&gt;While a 20MB transaction &lt;em&gt;could&lt;/em&gt; write over a gigabyte of data, I don&apos;t think that&apos;s necessarily a common occurrence.&#160;&lt;/p&gt;</comment>
                            <comment id="2762581" author="geert.bosch" created="Fri, 24 Jan 2020 17:43:01 +0000"  >&lt;p&gt;It is probably best to have the batcher always end the batch after a &amp;gt; 16 MB txn commit. This is never too pessimistic, because even the single &amp;gt; 16MB transaction is a reasonable amount of work. The current heuristics of ending a batch after either 5000 operations or 100MB worth of oplog, but even just 60 commits for a &amp;gt;16MB transaction can easily exceed 1 GB of data written.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10420">
                    <name>Backports</name>
                                            <outwardlinks description="backported by">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12450" key="com.atlassian.jira.plugin.system.customfieldtypes:multicheckboxes">
                        <customfieldname>Backport Requested</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="18953"><![CDATA[v4.4]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 24 Jan 2020 17:43:01 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        3 years, 48 weeks, 5 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>PM-1168</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            3 years, 48 weeks, 5 days ago
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_16465" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Linked BF Score</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>45.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>andrew.chen@mongodb.com</customfieldvalue>
            <customfieldvalue>schwerin@mongodb.com</customfieldvalue>
            <customfieldvalue>geert.bosch@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>maria.vankeulen@mongodb.com</customfieldvalue>
            <customfieldvalue>matthew.russotto@mongodb.com</customfieldvalue>
            <customfieldvalue>siyuan.zhou@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hwh85z:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr5f1b:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="3630">Execution Team 2020-02-24</customfieldvalue>
    <customfieldvalue id="3631">Execution Team 2020-03-09</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hwgufb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>