<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:47:04 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-37800] ChangeStream performance issues</title>
                <link>https://jira.mongodb.org/browse/SERVER-37800</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;For last 3 monthes our team has issues with MongoDB ChangeStream performance.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Numbers that we want to achieve - stable 15k documents/sec for 3 ChangeStream consumers over independent collections/databases.&#160;&lt;/p&gt;

&lt;p&gt;Current performance numbers - about 7-8k for 1 ChangeStream consumer.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Question:&#160;How can we achieve desired performance numbers on current cluster setup?&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;MongoDB Cluster deployed on AWS and has following topology:&lt;/p&gt;

&lt;p&gt;&#160; Shards: 6 shards with PSA arcitecture. Primary/Secondary -&#160;m5.xlarge nodes with EBS disks (2TB of disk space and 6k IOPS for each shard), arbiter - t2.medium node&lt;/p&gt;

&lt;p&gt;&#160; Mongos: 3 m5.xlarge nodes, write load balanced across all of it&lt;/p&gt;

&lt;p&gt;&#160; Config server: replica set of 3 m5.large nodes&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Key points of our work:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;changing number of shards and proper collection sharding don&apos;t boost performance&lt;/li&gt;
	&lt;li&gt;increasing instance type for shards and disk shape has low impact on performance&lt;/li&gt;
	&lt;li&gt;monitoring for cluster don&apos;t show any kind of bottlenecks. CPU, memory, disk usage, networking values for shards and mongos are in norm&lt;/li&gt;
	&lt;li&gt;increasing document size degrade ChangeStream performance. We pass _id field only&lt;/li&gt;
	&lt;li&gt;increasing number of writes decrease ChangeStream performance: 8k+/sec writes - 8k/sec consume, 12k/sec writes - 5k/sec consume, 14k/sec writes - 3k/sec consume&lt;/li&gt;
	&lt;li&gt;tuning MongoDB server side settings don&apos;t improve performance: decreasing&#160;&lt;em&gt;waitForSecondaryBeforeNoopWriteMS,&lt;/em&gt; increasing of&#160;__&#160;&lt;em&gt;ShardingTaskExecutorPoolMaxConnecting&lt;/em&gt; and&#160;__&#160;&lt;em&gt;replWriterThreadCount&lt;/em&gt;&lt;/li&gt;
	&lt;li&gt;tuning client side settings don&apos;t improve performance: changing of batch size, number of connections, filter for inserts and updates&lt;/li&gt;
	&lt;li&gt;profiling of client side show large waiting time on acquiring of ChangeStream notification (detailed info in attachments)&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;</description>
                <environment></environment>
        <key id="626244">SERVER-37800</key>
            <summary>ChangeStream performance issues</summary>
                <type id="3" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14718&amp;avatarType=issuetype">Task</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="-1">Unassigned</assignee>
                                    <reporter username="Knyshau">Ihar Knyshau [X]</reporter>
                        <labels>
                    </labels>
                <created>Mon, 29 Oct 2018 17:22:49 +0000</created>
                <updated>Mon, 26 Nov 2018 03:05:19 +0000</updated>
                            <resolved>Mon, 29 Oct 2018 19:28:24 +0000</resolved>
                                    <version>4.0.0</version>
                                                    <component>Performance</component>
                                        <votes>0</votes>
                                    <watches>5</watches>
                                                                                                                <comments>
                            <comment id="2045409" author="daniel.hatcher" created="Mon, 29 Oct 2018 19:28:17 +0000"  >&lt;p&gt;Thanks for your report. Please note that SERVER project is for reporting bugs or feature suggestions for the MongoDB server. For MongoDB-related support discussion please post on the &lt;a href=&quot;https://groups.google.com/group/mongodb-user&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;mongodb-user group&lt;/a&gt; or &lt;a href=&quot;https://stackoverflow.com/questions/tagged/mongodb&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;Stack Overflow with the &lt;tt&gt;mongodb&lt;/tt&gt; tag&lt;/a&gt;. A question like this involving more discussion would be best posted on the mongodb-user group.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                                                <inwardlinks description="is duplicated by">
                                                        </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="199792" name="ChangeStreamProfiling_1.PNG" size="167004" author="Knyshau" created="Mon, 29 Oct 2018 16:53:20 +0000"/>
                            <attachment id="199793" name="ChangeStreamProfiling_2.PNG" size="160118" author="Knyshau" created="Mon, 29 Oct 2018 16:53:24 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 29 Oct 2018 19:28:17 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        5 years, 15 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>wan.bachtiar@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            5 years, 15 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>daniel.hatcher@mongodb.com</customfieldvalue>
            <customfieldvalue>Knyshau</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hubl5b:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hu1uxr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hub7en:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>