<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:09:49 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-45827] Expand initial sync fuzzer grammar to include all CRUD document shapes and index DDL ops</title>
                <link>https://jira.mongodb.org/browse/SERVER-45827</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;The initial sync &lt;a href=&quot;https://github.com/mongodb/mongo/blob/8002575d7d5af37ff48e5b32a36dd105f6cc75bd/buildscripts/resmokeconfig/suites/initial_sync_fuzzer.yml&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;fuzzer suites&lt;/a&gt; run a series of randomized operations against the sync source of an initial syncing node in a way that is highly deterministic and reproducible. The diversity of operation types that it runs, however, is low, since it relies on a &lt;a href=&quot;https://github.com/10gen/jstestfuzz/blob/113d015b6feaf2919fddb2c5b5927bd808f935e8/src/libfuzz/repl_ops.ne&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;simplified grammar&lt;/a&gt; for generating operations (both CRUD ops and DDL ops). To give us thorough coverage of initial sync bugs that require more complex operation types, we should extend the grammar of the initial sync fuzzer to include all document shapes we can reasonably include in the grammar, and also include create/drop index operations with all index shapes. This should give us more thorough and reproducible coverage of initial sync.&lt;/p&gt;

&lt;p&gt;Our existing &lt;tt&gt;jstestfuzz_replication_initsync&lt;/tt&gt; suites give us coverage of initial sync with good operation diversity, but those failures can be much harder to reproduce due to the inherent non-determinism of the &lt;a href=&quot;https://github.com/mongodb/mongo/blob/e4347d57e5e1d6e2090417d4560d713f2cd2743e/buildscripts/resmokelib/testing/hooks/initialsync.py#L18&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;BackgroundInitialSync&lt;/a&gt; test hook.&lt;/p&gt;</description>
                <environment></environment>
        <key id="1119146">SERVER-45827</key>
            <summary>Expand initial sync fuzzer grammar to include all CRUD document shapes and index DDL ops</summary>
                <type id="4" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14710&amp;avatarType=issuetype">Improvement</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="10038" iconUrl="https://jira.mongodb.org/images/icons/subtask.gif" description="">Backlog</status>
                    <statusCategory id="2" key="new" colorName="default"/>
                                    <resolution id="-1">Unresolved</resolution>
                                        <assignee username="backlog-server-repl">Backlog - Replication Team</assignee>
                                    <reporter username="william.schultz@mongodb.com">William Schultz</reporter>
                        <labels>
                            <label>former-quick-wins</label>
                    </labels>
                <created>Tue, 28 Jan 2020 20:05:50 +0000</created>
                <updated>Tue, 6 Dec 2022 02:37:13 +0000</updated>
                                                                            <component>Replication</component>
                    <component>Testing Infrastructure</component>
                                        <votes>0</votes>
                                    <watches>7</watches>
                                                                                                                <comments>
                            <comment id="2842163" author="william.schultz" created="Fri, 7 Feb 2020 20:44:48 +0000"  >&lt;p&gt;After discussion with &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=robert.guo&quot; class=&quot;user-hover&quot; rel=&quot;robert.guo&quot;&gt;robert.guo&lt;/a&gt; and &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=judah.schvimer&quot; class=&quot;user-hover&quot; rel=&quot;judah.schvimer&quot;&gt;judah.schvimer&lt;/a&gt;, we have determined that running the mutational fuzzer against the initial sync fuzzer test fixture will require too much work for a quick win. Instead, we&apos;ve decided to extend the existing initial sync fuzzer grammar to include complex document and index shapes. It may also be possible to combine the initial sync and rollback fuzzer grammars. We still think that running mutational fuzzer ops against the initial sync fuzzer suite is valuable, so I have created &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-46044&quot; title=&quot;Run mutational fuzzer operations against sync source in initial sync fuzzer suites&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-46044&quot;&gt;SERVER-46044&lt;/a&gt; to keep track of that future work.&lt;/p&gt;</comment>
                            <comment id="2781499" author="robert.guo" created="Mon, 3 Feb 2020 22:44:05 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=william.schultz&quot; class=&quot;user-hover&quot; rel=&quot;william.schultz&quot;&gt;william.schultz&lt;/a&gt; I&apos;m not optimistic the grammar fuzzers will work out well. The grammar fuzzers are for enumerating possible known states that a human can think of; the mutational fuzzer is for generating states people do not think of, like &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-32903&quot; title=&quot;Ambiguous field name error should be ignored during initial sync&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-32903&quot;&gt;&lt;del&gt;SERVER-32903&lt;/del&gt;&lt;/a&gt;. It&apos;s hard to encode what we don&apos;t know into a grammar without sacrificing validity rating.&lt;/p&gt;

&lt;p&gt;One possible grammar-based solution is to write code to mutate the grammar rules to generate new rules. We can then gather coverage data to decide if the new, generated rule is any good. This allows the goal of incrementally increasing the complexity of the grammar over time, as you said. Without coverage info, we won&apos;t know if any change to the grammar that knowingly introduces invalid queries does more harm or good.&lt;/p&gt;</comment>
                            <comment id="2781400" author="william.schultz" created="Mon, 3 Feb 2020 21:54:54 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=robert.guo&quot; class=&quot;user-hover&quot; rel=&quot;robert.guo&quot;&gt;robert.guo&lt;/a&gt; Even though this ticket&apos;s title says that we want to run the mutational fuzzer operations in the initial sync fuzzer, the underlying goal is to sufficiently increase the diversity of operations run in the initial sync fuzzer. Do you think that this could be achieved more easily by taking advantage of our other existing, more advanced grammars? Extending a single grammar file or utilizing other existing grammars seems much easier than integrating the mutational fuzzer with the initial sync fuzzer (based on the difficulties you outlined above). I imagine the complexity of those grammars can also be incrementally increased over time. &lt;/p&gt;</comment>
                            <comment id="2781365" author="judah.schvimer" created="Mon, 3 Feb 2020 21:41:59 +0000"  >&lt;p&gt;Thanks for the input. I think the value of this proposed work (and maybe a similar thing for the rollback fuzzer) is significantly more valuable than making the background init sync hook more deterministic. I think a lot of the value from this work comes from the initial sync fuzzer&apos;s ability to run commands in narrow windows to expose difficult race conditions. This is unrelated to the additional determinism. I propose moving this from a replication quick-win to a dev-prod QP request. Does that seem reasonable?&lt;/p&gt;</comment>
                            <comment id="2781116" author="robert.guo" created="Mon, 3 Feb 2020 20:02:36 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=judah.schvimer&quot; class=&quot;user-hover&quot; rel=&quot;judah.schvimer&quot;&gt;judah.schvimer&lt;/a&gt; While the idea in this ticket is sound and valuable, I expect it to be a lot of work. The complexity comes from the need to &quot;control&quot; what the mutational fuzzer does to allow initsync to happen &quot;inline&quot; instead of &quot;in the background&quot;. The rough breakdown of work items is as follows:&lt;/p&gt;

&lt;p&gt;1. Parse the fuzzer generated file and split it into sub-ASTs that can be individually run in a single initsync fixture step. Since the mongo shell has no proper module support, we need to somehow prevent variables used by the intisync fixture from being overridden.&lt;br/&gt;
2. Repl commands are similarly blacklisted from the mutational fuzzer, we need to somehow prevent them from being blacklisted, but only when they are run from the initsync fixture, not the rest of the mutational fuzzer. One way is to replace the variable names in the initsync fixure with UIDs that won&apos;t appear in the rest of the tests, this has to happen at runtime because the mutational fuzzer can read the initsync fixture file as input to be mutated.&lt;br/&gt;
3. Merge the initsync fixture commands into the generated file; in the grammar fuzzers, this is done through a template and placeholders defined in the grammar; the mutational fuzzer will likely have to use AST parsing.&lt;/p&gt;

&lt;p&gt;Before trying to make a decision on the above work, I&apos;m curious if we&apos;ve thought about making background init sync hook more deterministic. Could we maybe use change streams and the failpoint from &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-45830&quot; title=&quot;Add failpoint to allow InitialSyncTest fixture to pause initial syncing node after cloning some documents&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-45830&quot;&gt;SERVER-45830&lt;/a&gt; to have the background hook execute a deterministic number of statements between each initsync step? I suspect this approach will be significantly less work if it&apos;s feasible&lt;/p&gt;</comment>
                            <comment id="2780795" author="tess.avitabile" created="Mon, 3 Feb 2020 18:20:59 +0000"  >&lt;p&gt;After we do this and&#160;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-45830&quot; title=&quot;Add failpoint to allow InitialSyncTest fixture to pause initial syncing node after cloning some documents&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-45830&quot;&gt;SERVER-45830&lt;/a&gt;, we would attempt to remove&#160;jstestfuzz_replication_initsync. We could let them both run for a month to make sure they&apos;re catching the same bugs.&lt;/p&gt;</comment>
                            <comment id="2774217" author="william.schultz" created="Fri, 31 Jan 2020 00:49:27 +0000"  >&lt;p&gt;Another thought about improving the initial sync fuzzer incrementally: when we fix a particular bug whose reproduction requires certain shapes of operations, we could consider adding these new operation types into the initial sync fuzzer grammar. For example, we worked for a day or two to reproduce the original bug in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-32903&quot; title=&quot;Ambiguous field name error should be ignored during initial sync&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-32903&quot;&gt;&lt;del&gt;SERVER-32903&lt;/del&gt;&lt;/a&gt;, but if we had added the relevant operation types to the fuzzer grammar, I imagine that the fuzzer could have discovered a deterministic repro for us automatically. Just a thought for when we are fixing future, obscure initial sync bugs that depend on specific operation types or document shapes.&lt;/p&gt;</comment>
                            <comment id="2769092" author="william.schultz" created="Tue, 28 Jan 2020 20:06:56 +0000"  >&lt;p&gt;This type of coverage would presumably allow bugs like&#160;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-32903&quot; title=&quot;Ambiguous field name error should be ignored during initial sync&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-32903&quot;&gt;&lt;del&gt;SERVER-32903&lt;/del&gt;&lt;/a&gt; to be caught by the initial sync fuzzer.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="487436">SERVER-32903</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1119195">SERVER-45830</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="1139482">SERVER-46044</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>8.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_12751" key="com.atlassian.jira.plugin.system.customfieldtypes:multiselect">
                        <customfieldname>Assigned Teams</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="25128"><![CDATA[Replication]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 31 Jan 2020 12:44:08 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        4 years, 5 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>alexander.golin@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            4 years, 5 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>backlog-server-repl</customfieldvalue>
            <customfieldvalue>judah.schvimer@mongodb.com</customfieldvalue>
            <customfieldvalue>robert.guo@mongodb.com</customfieldvalue>
            <customfieldvalue>tess.avitabile@mongodb.com</customfieldvalue>
            <customfieldvalue>william.schultz@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hwlwkf:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr5o8n:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hwlitr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>