<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:07:56 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-24978] Second batches in aggregation framework are asked synchronously</title>
                <link>https://jira.mongodb.org/browse/SERVER-24978</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;In the case of an aggregation framework query the first batch of results asked from the &lt;tt&gt;mergerPart&lt;/tt&gt; to the shards are done asynchronously, however when it needs to get a second batch to fullfill the query it is virtually mono-threaded as the call to &lt;tt&gt;getNext&lt;/tt&gt; is synchronous.&lt;/p&gt;

&lt;p&gt;I think it would be good to be able to overcome this limitation and make all subsequent fetches to be done in the background.&lt;/p&gt;

&lt;p&gt;You can see here (&lt;a href=&quot;https://github.com/ahom/jupyter-notebooks/blob/master/mongo_cs32044/notebook.ipynb&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/ahom/jupyter-notebooks/blob/master/mongo_cs32044/notebook.ipynb&lt;/a&gt;) a representation of this behavior.&lt;/p&gt;

&lt;p&gt;On the Y axis are the shards, on the X axis is the time. Starting from second batch seems is completely synchronous.&lt;/p&gt;

&lt;p&gt;Cheers,&lt;br/&gt;
Antoine&lt;/p&gt;</description>
                <environment></environment>
        <key id="299978">SERVER-24978</key>
            <summary>Second batches in aggregation framework are asked synchronously</summary>
                <type id="4" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14710&amp;avatarType=issuetype">Improvement</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="charlie.swanson@mongodb.com">Charlie Swanson</assignee>
                                    <reporter username="antoine.hom@amadeus.com">Antoine Hom</reporter>
                        <labels>
                    </labels>
                <created>Mon, 11 Jul 2016 12:59:53 +0000</created>
                <updated>Fri, 20 Apr 2018 14:28:34 +0000</updated>
                            <resolved>Tue, 6 Mar 2018 17:26:15 +0000</resolved>
                                                    <fixVersion>3.7.3</fixVersion>
                                    <component>Aggregation Framework</component>
                                        <votes>0</votes>
                                    <watches>10</watches>
                                                                                                                <comments>
                            <comment id="1824742" author="charlie.swanson" created="Tue, 6 Mar 2018 17:26:15 +0000"  >&lt;p&gt;The recently committed fix should mitigate the described problem. Before, the $mergeCursors stage would getMore each cursor, one by one, waiting for a response from one cursor before iterating the next. Now each getMore will be scheduled independently through an asynchronous interface, which should improve the performance here.&lt;/p&gt;</comment>
                            <comment id="1824738" author="xgen-internal-githook" created="Tue, 6 Mar 2018 17:23:33 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{&apos;email&apos;: &apos;charlie.swanson@mongodb.com&apos;, &apos;name&apos;: &apos;Charlie Swanson&apos;, &apos;username&apos;: &apos;cswanson310&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-24978&quot; title=&quot;Second batches in aggregation framework are asked synchronously&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-24978&quot;&gt;&lt;del&gt;SERVER-24978&lt;/del&gt;&lt;/a&gt; Use AsyncResultsMerger in $mergeCursors&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/a7106b407cecdcfa8ba6c8765c9874bce65a6d5a&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/a7106b407cecdcfa8ba6c8765c9874bce65a6d5a&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1823111" author="charlie.swanson" created="Mon, 5 Mar 2018 15:43:23 +0000"  >&lt;p&gt;A patch to fix this has made it through code review, but exposed an issue tracked in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-33660&quot; title=&quot;Once getMores include lsid, sharded aggregations with $mergeCursors can hang&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-33660&quot;&gt;&lt;del&gt;SERVER-33660&lt;/del&gt;&lt;/a&gt;.&lt;/p&gt;</comment>
                            <comment id="1751215" author="david.storch" created="Wed, 13 Dec 2017 19:21:48 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=charlie.swanson&quot; class=&quot;user-hover&quot; rel=&quot;charlie.swanson&quot;&gt;charlie.swanson&lt;/a&gt; pointed out that it would be easier to replace DocumentSourceMergeCursors with the AsyncResultsMerger once we have implemented &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-32307&quot; title=&quot;Make AsyncResultsMerger kill sequence issue killCursors without waiting for outstanding batches&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-32307&quot;&gt;&lt;del&gt;SERVER-32307&lt;/del&gt;&lt;/a&gt;. Therefore, I&apos;m adding a &quot;depends on&quot; link and unscheduling this ticket. We should start on it as soon as &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-32307&quot; title=&quot;Make AsyncResultsMerger kill sequence issue killCursors without waiting for outstanding batches&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-32307&quot;&gt;&lt;del&gt;SERVER-32307&lt;/del&gt;&lt;/a&gt; is completed.&lt;/p&gt;</comment>
                            <comment id="1749970" author="charlie.swanson" created="Tue, 12 Dec 2017 18:29:30 +0000"  >&lt;p&gt;I think one good way to do this would be to replace the merging machinery within DocumentSourceMergeCursors to utilize the &lt;a href=&quot;https://github.com/mongodb/mongo/blob/1cb47465e4d460a0a76ba9985d6adec4bc09cb38/src/mongo/s/query/async_results_merger.h#L74&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;AsyncResultsMerger&lt;/a&gt; utility which is used on mongos. This should resolve this issue since the AsyncResultsMerger schedules the getMores independently of waiting for the responses, so can schedule multiple getMores at a time.&lt;/p&gt;</comment>
                            <comment id="1320769" author="charlie.swanson" created="Mon, 11 Jul 2016 22:17:09 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=david.storch&quot; class=&quot;user-hover&quot; rel=&quot;david.storch&quot;&gt;david.storch&lt;/a&gt;, we had some discussion earlier about the strategy of issuing the getMore requests. It looks like the cursor merging initializes with the &apos;better&apos; strategy of issuing all requests, then waiting for all responses:&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/blob/r3.3.9/src/mongo/db/pipeline/document_source_merge_cursors.cpp#L112-L134&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/blob/r3.3.9/src/mongo/db/pipeline/document_source_merge_cursors.cpp#L112-L134&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;But that it&apos;s not smart at all about issuing getMores:&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/blob/r3.3.9/src/mongo/db/pipeline/document_source_merge_cursors.cpp#L136-L169&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/blob/r3.3.9/src/mongo/db/pipeline/document_source_merge_cursors.cpp#L136-L169&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;It looks like this will just block whenever you happen to exhaust a batch, which probably is exactly the same time for each cursor.&lt;/p&gt;

&lt;p&gt;I&apos;m not sure how easy this will be to fix though, since I&apos;m not comfortable with relying on all cursors having the same batch size. In particular, I think it should be fine if there was a batchSize specified to the aggregation, but I&apos;m worried about doing such an optimization for a default batch size. In particular, if we ever change the default batch size, then things could go awry in a mixed-version cluster.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                            <outwardlinks description="depends on">
                                        <issuelink>
            <issuekey id="505762">SERVER-33660</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="471776">SERVER-32307</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is depended on by">
                                        <issuelink>
            <issuekey id="496860">SERVER-33280</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="497821">SERVER-33323</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                                                <inwardlinks description="is duplicated by">
                                        <issuelink>
            <issuekey id="460313">SERVER-32014</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="299977">SERVER-24981</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="130249" name="getmore.png" size="119758" author="antoine.hom@amadeus.com" created="Mon, 11 Jul 2016 13:00:58 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>6.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>4.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[500A000000UcXY3IAN]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 11 Jul 2016 22:17:09 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        5 years, 49 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[<s><a href='https://jira.mongodb.org/browse/SERVER-32307'>SERVER-32307</a></s>, <s><a href='https://jira.mongodb.org/browse/SERVER-33660'>SERVER-33660</a></s>]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                            <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>PM-943</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>charlie.swanson@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            5 years, 49 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_16465" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Linked BF Score</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>antoine.hom@amadeus.com</customfieldvalue>
            <customfieldvalue>charlie.swanson@mongodb.com</customfieldvalue>
            <customfieldvalue>david.storch@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrk34f:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr8y2n:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="2096">Query 2018-01-29</customfieldvalue>
    <customfieldvalue id="2117">Query 2018-02-12</customfieldvalue>
    <customfieldvalue id="2120">Query 2018-02-26</customfieldvalue>
    <customfieldvalue id="2121">Query 2018-03-12</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrj6xj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>