<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 06:38:43 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-78580] Improve $project parsing to avoid quadratic behavior</title>
                <link>https://jira.mongodb.org/browse/SERVER-78580</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;ProjectionPathASTNode &lt;a href=&quot;https://github.com/10gen/mongo/blob/cc4bc680c434cddaa2c5652680cab97ebc6f5889/src/mongo/db/query/projection_ast.h#L184-L192&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;getChild&lt;/a&gt; performs a linear search through a vector of field names to find the correct child. Because of this search, we see n^2 behavior on the number of fields in the $project. As we add children to this node in &lt;a href=&quot;https://github.com/10gen/mongo/blob/master/src/mongo/db/query/projection_parser.cpp#L178-L186&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;addNodeAtPathHelper&lt;/a&gt;, we also call getChild to search through everything we&apos;ve added so far.&lt;/p&gt;

&lt;p&gt;ProjectionPathASTNode could use a map instead of a vector to avoid this behavior.&lt;/p&gt;

&lt;p&gt;This ticket should also involve performance testing with a map. If we see a regression, we could consider using a map when there are &amp;gt;50 (or some N) elements.&lt;/p&gt;</description>
                <environment></environment>
        <key id="2381659">SERVER-78580</key>
            <summary>Improve $project parsing to avoid quadratic behavior</summary>
                <type id="4" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14710&amp;avatarType=issuetype">Improvement</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13201">Fixed</resolution>
                                        <assignee username="matt.boros@mongodb.com">Matt Boros</assignee>
                                    <reporter username="matt.boros@mongodb.com">Matt Boros</reporter>
                        <labels>
                    </labels>
                <created>Fri, 30 Jun 2023 18:04:19 +0000</created>
                <updated>Sun, 29 Oct 2023 21:19:20 +0000</updated>
                            <resolved>Mon, 14 Aug 2023 21:35:47 +0000</resolved>
                                                    <fixVersion>7.1.0-rc0</fixVersion>
                                                        <votes>0</votes>
                                    <watches>6</watches>
                                                                                                                <comments>
                            <comment id="5616151" author="xgen-internal-githook" created="Mon, 7 Aug 2023 02:40:09 +0000"  >&lt;p&gt;Author: &lt;/p&gt;
{&apos;name&apos;: &apos;Matt Boros&apos;, &apos;email&apos;: &apos;matt.boros@mongodb.com&apos;, &apos;username&apos;: &apos;mattBoros&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-78580&quot; title=&quot;Improve $project parsing to avoid quadratic behavior&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-78580&quot;&gt;&lt;del&gt;SERVER-78580&lt;/del&gt;&lt;/a&gt; Improve $project parsing to avoid quadratic behavior&lt;br/&gt;
Branch: minh.luu-no_compile_sys-perf&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/196f02229d587c1e1212a3a21c6c670f29d75839&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/196f02229d587c1e1212a3a21c6c670f29d75839&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="5611665" author="xgen-internal-githook" created="Thu, 3 Aug 2023 20:46:03 +0000"  >&lt;p&gt;Author: &lt;/p&gt;
{&apos;name&apos;: &apos;Matt Boros&apos;, &apos;email&apos;: &apos;matt.boros@mongodb.com&apos;, &apos;username&apos;: &apos;mattBoros&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-78580&quot; title=&quot;Improve $project parsing to avoid quadratic behavior&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-78580&quot;&gt;&lt;del&gt;SERVER-78580&lt;/del&gt;&lt;/a&gt; Improve $project parsing to avoid quadratic behavior&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/196f02229d587c1e1212a3a21c6c670f29d75839&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/196f02229d587c1e1212a3a21c6c670f29d75839&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="5606305" author="xgen-internal-githook" created="Tue, 1 Aug 2023 22:05:16 +0000"  >&lt;p&gt;Author: &lt;/p&gt;
{&apos;name&apos;: &apos;Matthew Boros&apos;, &apos;email&apos;: &apos;mattBoros@users.noreply.github.com&apos;, &apos;username&apos;: &apos;mattBoros&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-78580&quot; title=&quot;Improve $project parsing to avoid quadratic behavior&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-78580&quot;&gt;&lt;del&gt;SERVER-78580&lt;/del&gt;&lt;/a&gt; projection parsing workload (#956)&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/genny/commit/3ca962077cb08b0f42b5e8eeba13616c5b66666a&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/genny/commit/3ca962077cb08b0f42b5e8eeba13616c5b66666a&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="5596906" author="JIRAUSER1253388" created="Thu, 27 Jul 2023 21:14:14 +0000"  >&lt;p&gt;&lt;a href=&quot;https://github.com/mongodb/genny/pull/956&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/genny/pull/956&lt;/a&gt; for genny workload&lt;/p&gt;</comment>
                            <comment id="5577372" author="JIRAUSER1253388" created="Wed, 19 Jul 2023 16:23:08 +0000"  >&lt;p&gt;Decided that &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-78631&quot; title=&quot;[CQF] Replace linear search in BoolExpr::Builder&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-78631&quot;&gt;&lt;del&gt;SERVER-78631&lt;/del&gt;&lt;/a&gt; could add this generic structure, because it doesn&apos;t fit cleanly into this ticket&apos;s scope. I&apos;ll post the code I had for this vector/map structure in a comment under &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-78631&quot; title=&quot;[CQF] Replace linear search in BoolExpr::Builder&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-78631&quot;&gt;&lt;del&gt;SERVER-78631&lt;/del&gt;&lt;/a&gt;.&lt;/p&gt;</comment>
                            <comment id="5575778" author="JIRAUSER1253388" created="Wed, 19 Jul 2023 00:49:55 +0000"  >&lt;p&gt;My plan for this was to create a generic map class templated on the key/value types and the type of map to use when the number of elements is greater than some threshold. Until that threshold is hit, a vector would be used. This would solve the root problem of this ticket which is ProjectionPathASTNode. It could also be used for &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-78631&quot; title=&quot;[CQF] Replace linear search in BoolExpr::Builder&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-78631&quot;&gt;&lt;del&gt;SERVER-78631&lt;/del&gt;&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;However ProjectionPathASTNode needs the inserted data to be kept in order, which is possible with a hash map where the keys are part of a linked list. This is not a data structure provided by the standard library. Our preferred way to get the linked hash map behavior is to use &lt;a href=&quot;https://github.com/mongodb/mongo/blob/master/src/mongo/logv2/log_severity_suppressor.h#L88-L93&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;a boost::multi_index_container&lt;/a&gt; which does not have the same interface as a regular map, and therefore wouldn&apos;t work with the generic container I was thinking of creating. Similarly, the &lt;a href=&quot;https://github.com/10gen/mongo/blob/6aa81b7a5d8e5cc634e35f24a0b98efc3a83b865/src/mongo/util/lru_cache.h#L311-L317&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;LRU Cache implementation&lt;/a&gt; we have maintains its own data structures rather than using a linked hash map.&lt;/p&gt;

&lt;p&gt;I could&lt;br/&gt;
1. Give up on creating this generic data structure and fix the problem non-generically.&lt;br/&gt;
2. Write my own LinkedHashMap on top of an existing c++ map structure, then refactor lru_cache.h and the generic map structure I described to use this.&lt;br/&gt;
3. Find a library that provides a linked hash map. This would be excessive.&lt;/p&gt;

&lt;p&gt;I think I will go with #2 but will confirm with someone first.&lt;/p&gt;</comment>
                            <comment id="5558753" author="JIRAUSER1253388" created="Tue, 11 Jul 2023 20:31:43 +0000"  >&lt;p&gt;My plan is to use a vector for small projections and when enough nodes are added we switch over to a map. I did some investigation and found n=100 to be a good crossover point. Unfortunately there aren&apos;t any jstests in agg/ or core/ that test for this size projection so I&apos;ll add one. I&apos;m considering adding a Genny workload as well. &lt;/p&gt;

&lt;p&gt;Ran a &lt;a href=&quot;https://performance-analyzer.server-tig.prod.corp.mongodb.com/perf-analyzer-viz/?comparison_id=7e8d063d-fdc7-4757-adcd-9fad8399ed69&amp;amp;determination_filter=Regressed&amp;amp;percent_filter=0%7C%7C100&amp;amp;z_filter=0%7C%7C10&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;perf patch&lt;/a&gt; with workloads we currently have, and as expected it&apos;s too noisy to make conclusions. I&apos;ll run another patch and test locally to see what&apos;s going on.&lt;/p&gt;

&lt;p&gt;Here is what the curve looks like with the fix.&lt;br/&gt;
&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;a id=&quot;461724_thumb&quot; href=&quot;https://jira.mongodb.org/secure/attachment/461724/461724_figure_with_fix.png&quot; title=&quot;figure_with_fix.png&quot; file-preview-type=&quot;image&quot; file-preview-id=&quot;461724&quot; file-preview-title=&quot;figure_with_fix.png&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/thumbnail/461724/_thumb_461724.png&quot; style=&quot;border: 0px solid black&quot; role=&quot;presentation&quot;/&gt;&lt;/a&gt;&lt;/span&gt; &lt;/p&gt;</comment>
                            <comment id="5550531" author="JIRAUSER1253388" created="Fri, 7 Jul 2023 14:30:12 +0000"  >&lt;p&gt;Thought about this and I don&apos;t think this ticket belongs in the Bonsai perf project. Will move it out later.&lt;/p&gt;</comment>
                            <comment id="5537563" author="JIRAUSER1253388" created="Fri, 30 Jun 2023 18:05:45 +0000"  >&lt;p&gt;For the largest $projects you can create, of the form {$project: {a: 1, b: 1, c: 1, ...}}, this currently takes 30+ minutes to parse. With a proof of concept I have, it takes less than a minute.&lt;/p&gt;

&lt;p&gt;Proof of concept: &lt;a href=&quot;https://github.com/10gen/mongo/compare/master...project-parse-fast&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/10gen/mongo/compare/master...project-parse-fast&lt;/a&gt;&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10520">
                    <name>Problem/Incident</name>
                                            <outwardlinks description="causes">
                                                        </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="1961583">SERVER-62509</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="461724" name="figure_with_fix.png" size="33279" author="matt.boros@mongodb.com" created="Tue, 11 Jul 2023 20:31:25 +0000"/>
                            <attachment id="460602" name="screenshot-1.png" size="34517" author="matt.boros@mongodb.com" created="Thu, 6 Jul 2023 19:20:12 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>3.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_12751" key="com.atlassian.jira.plugin.system.customfieldtypes:multiselect">
                        <customfieldname>Assigned Teams</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="25126"><![CDATA[Query Optimization]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Thu, 27 Jul 2023 19:45:55 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        26 weeks, 3 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            26 weeks, 3 days ago
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_16465" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Linked BF Score</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>35.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>matt.boros@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i2g66n:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|i1ybyw:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                            <customfield id="customfield_22250" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Special Downgrade Instructions Required</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="23343"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="7241">QE 2023-07-24</customfieldvalue>
    <customfieldvalue id="7243">QE 2023-08-07</customfieldvalue>
    <customfieldvalue id="7246">QE 2023-08-21</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i2fsbz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>