<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 06:53:43 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-83980] Investigate using callback gRPC API</title>
                <link>https://jira.mongodb.org/browse/SERVER-83980</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;When doing initial performance testing in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-80572&quot; title=&quot;Generate initial performance benchmarks via benchrun&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-80572&quot;&gt;&lt;del&gt;SERVER-80572&lt;/del&gt;&lt;/a&gt;, we found that gRPC was a lot slower than ASIO, and saw through flamegraphs that a lot of that slowness was due to gRPC acquiring a mutex shared by the whole process to synchronize access to the completion queue.&lt;/p&gt;

&lt;p&gt;After looking around, we saw on the &lt;a href=&quot;https://grpc.io/docs/guides/performance/#c&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;gRPC performance page&lt;/a&gt; that using the sync server is not suggested for performance-sensitive applications. gRPC instead provides a callback API, that is supposed to be easier to use than the async API but faster than the sync API.&lt;/p&gt;

&lt;p&gt;Investigate the changes required to implement the callback api instead of the sync api, and run initial performance benchmarks on them.&lt;/p&gt;</description>
                <environment></environment>
        <key id="2519205">SERVER-83980</key>
            <summary>Investigate using callback gRPC API</summary>
                <type id="3" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14718&amp;avatarType=issuetype">Task</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="erin.mcnulty@mongodb.com">Erin McNulty</assignee>
                                    <reporter username="erin.mcnulty@mongodb.com">Erin McNulty</reporter>
                        <labels>
                    </labels>
                <created>Thu, 7 Dec 2023 18:23:36 +0000</created>
                <updated>Fri, 5 Jan 2024 16:56:00 +0000</updated>
                            <resolved>Mon, 11 Dec 2023 16:04:24 +0000</resolved>
                                                                                        <votes>0</votes>
                                    <watches>2</watches>
                                                                                                                <comments>
                            <comment id="5940774" author="JIRAUSER1260475" created="Mon, 11 Dec 2023 16:03:53 +0000"  >&lt;p&gt;Update: it was easy to change the threading model using the method I described above &lt;img class=&quot;emoticon&quot; src=&quot;https://jira.mongodb.org/images/icons/emoticons/smile.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt; I was able to build and connect to the server using the callback gRPC API, and run some benchrun trials with it.&lt;/p&gt;

&lt;p&gt;TLDR is that we did not see significant improvements for gRPC with the callback API, but I would caution against using this data to justify ignoring the callback API for future investigations, because this was a very rough POC without any optimizations implemented.&lt;/p&gt;

&lt;p&gt;The results are summarized here:&lt;/p&gt;
&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;num threads&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&#160;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;with sync API&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;with callback API&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;gRPC Latency / MongoRPC Latency&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1.97&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;2.06&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;1&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;gRPC ops/s / MongoRPC ops/s&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;0.51&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;0.49&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&#160;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&#160;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&#160;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;&#160;&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;4&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;gRPC Latency / MongoRPC Latency&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;3.10&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;3.71&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;4&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;gRPC ops/s / MongoRPC ops/s&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;0.32&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;0.27&lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;


&lt;p&gt;As seen above, the callback API was not faster in initial performance runs with respect to MongoRPC on 1 or 4 threads-- it is around 2x slower on one thread for both implementations, and around 3-4x slower with 4 threads, with the sync API actually winning out by a bit. The &lt;a href=&quot;https://docs.google.com/spreadsheets/d/17lD9jwTe_BnLlIPEvY1r-28eJs_YlnLT0ek55IQy5AA/edit#gid=1845359524&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;more detailed results are here&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://github.com/10gen/mongo/compare/master...grpc-perf-testing?expand=1&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;POC branch&lt;/a&gt; is here, focus on the changes in the grpc/* folder. I am putting this down in order to focus on completing the correctness work for this project, but I think the next steps for PM-3366 are to:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;Profile this build, and try to note what exactly is happening in the session workflow that is slowing us down wrt ASIO. I think a good place to start might just be putting ScopedTimers throughout the session workflow.&lt;/li&gt;
	&lt;li&gt;Use the driver POCs to profile and evaluate this build-- the egress end of things in this POC is still using the sync API. Try to isolate the server ingress performance from the egress performance in order to gain a more complete understanding of where the gRPC slowness is.&lt;/li&gt;
	&lt;li&gt;Do some research about how other performance-oriented applications implement the gRPC callback API. I think taking the callback API and doing some research with that could put us in a better position to improve the performance of our gRPC implementation because the sync implementation is explicitly marked as not providing strong perf, while the callback API is supposed to be performant.&lt;/li&gt;
	&lt;li&gt;Evaluate the threading model in this POC-- right now, I just spin up another thread to do the handleStream() workflow on, which essentially mimics the thread per client model that MongoRPC implements. I think placing the threading model back in our hands could make investigating performance a lot easier, because for very little code we are relying on our synchronization primitives instead of gRPC&apos;s, which we have a better intuition for.&lt;/li&gt;
&lt;/ul&gt;
</comment>
                            <comment id="5933736" author="JIRAUSER1260475" created="Thu, 7 Dec 2023 18:24:42 +0000"  >&lt;p&gt;I was able to make changes so that gRPC was using the callback API, but I think that changing the threading model is not as simple as we thought. I ran into a segfault when connecting to the server, and the backtrace led me to the spot in&#160;&lt;a href=&quot;https://github.com/10gen/mongo/blob/ce1d3987d0b8b73e00f02a1117bc147539de6bf9/src/mongo/transport/session_workflow.cpp#L638&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;session_workflow&lt;/a&gt; right before we call source message. I also saw that right before this message, we switched threads. When I tried out the inline model with this, just to see what would happen, it locked up the entire server and I had to kill it.&lt;/p&gt;

&lt;p&gt;TLDR is that its easy enough to switch to the callback API in terms of our direct gRPC code, but the threading model might not be so simple. I think my next step might be to try to give each gRPC stream its own thread on the handleStream level, and then still use kInline when we enter into the session workflow. All of my changes are&#160;&lt;a href=&quot;https://github.com/10gen/mongo/compare/grpc-perf-testing...grpc-async-experiment?expand=1&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;here&lt;/a&gt;&#160;(this is from all of my perf testing, so only focus on the changes in the&#160;&lt;tt&gt;grpc/*&lt;/tt&gt;&#160;folder if you are interested in looking).&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        8 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_17050" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Downstream Team Attention</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="16941"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>PM-2923</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>erin.mcnulty@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            8 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>erin.mcnulty@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i33os7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|i2lgmc:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                            <customfield id="customfield_22250" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Special Downgrade Instructions Required</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="23343"><![CDATA[Not Needed]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="7936">Service Arch 2023-12-11</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|i33axj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>