<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:42:40 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-36291] Severe Performance regression from 3.4 to 3.6 w/ faulty logging or secondaryPreferred query</title>
                <link>https://jira.mongodb.org/browse/SERVER-36291</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;When running our systems on 3.6 vs 3.4 with the exact same data, we&apos;re seeing intermittent slowness happening with slow queries logging as secondaryPreferred, while not a single one of our queries is a against a secondary. We only do primaryPreferred queries.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;See my attachments&#160;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-34518&quot; title=&quot;WT Transaction / Timestamp error during non-responsiveness&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-34518&quot;&gt;&lt;del&gt;SERVER-34518&lt;/del&gt;&lt;/a&gt;&lt;/p&gt;</description>
                <environment></environment>
        <key id="576373">SERVER-36291</key>
            <summary>Severe Performance regression from 3.4 to 3.6 w/ faulty logging or secondaryPreferred query</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="dmitry.agranat@mongodb.com">Dmitry Agranat</assignee>
                                    <reporter username="sallgeud">Chad Kreimendahl</reporter>
                        <labels>
                    </labels>
                <created>Thu, 26 Jul 2018 01:34:22 +0000</created>
                <updated>Fri, 8 Mar 2019 10:03:07 +0000</updated>
                            <resolved>Wed, 6 Feb 2019 11:25:37 +0000</resolved>
                                    <version>3.6.6</version>
                                                    <component>Performance</component>
                                        <votes>2</votes>
                                    <watches>16</watches>
                                                                                                                <comments>
                            <comment id="2139739" author="sallgeud" created="Wed, 6 Feb 2019 17:05:55 +0000"  >&lt;p&gt;Definitely not yet resolved. We&apos;ll attempt the test with the config file change soon. Our only issue is that these types of issues are not easy to create outside the demands of production. Recreating production issues in dev is nearly impossible.&lt;/p&gt;</comment>
                            <comment id="2139327" author="dmitry.agranat" created="Wed, 6 Feb 2019 11:25:24 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sallgeud&quot; class=&quot;user-hover&quot; rel=&quot;sallgeud&quot;&gt;sallgeud&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;We haven&#8217;t heard back from you for some time, so I&#8217;m going to mark this ticket as resolved. If this is still an issue for you, please provide additional information and we will reopen the ticket.&lt;/p&gt;

&lt;p&gt;Regards,&lt;/p&gt;</comment>
                            <comment id="2102315" author="dmitry.agranat" created="Mon, 31 Dec 2018 08:42:09 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sallgeud&quot; class=&quot;user-hover&quot; rel=&quot;sallgeud&quot;&gt;sallgeud&lt;/a&gt;,&lt;/p&gt;

&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;a small proportion: instances that have in excess of 20K active collections and indexes.&lt;/li&gt;
	&lt;li&gt;constantly access: means being queried or updated more than once every 5 minutes&lt;/li&gt;
&lt;/ul&gt;


&lt;blockquote&gt;
&lt;p&gt;Unfortunately, sharding is nearly pointless in our environment, as &amp;gt; 90% of customers will never really use more than 50GB &lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;This might be a good use case for micro-sharding, you can co-locate MongoDB processes on the same server, by isolating processes and their overall resource utilization by using a container, for example Docker or cgroups.&lt;/p&gt;

&lt;p&gt;Thank you,&lt;br/&gt;
Dima&lt;/p&gt;</comment>
                            <comment id="2101124" author="sallgeud" created="Thu, 27 Dec 2018 18:49:20 +0000"  >&lt;p&gt;What&apos;s considered a &quot;small proportion&quot;?&#160; &#160;Let&apos;s say we have 350k &quot;tables&quot;, and we regularly access about 25-50k of them.&#160; What about regularly accessing 10k vs 100k of them, compared to 25-50k?&lt;/p&gt;

&lt;p&gt;Also, what is &quot;consistently access&quot;? Hourly, daily, etc?&lt;/p&gt;</comment>
                            <comment id="2100825" author="dmitry.agranat" created="Thu, 27 Dec 2018 12:14:19 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sallgeud&quot; class=&quot;user-hover&quot; rel=&quot;sallgeud&quot;&gt;sallgeud&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;To set the mentioned WT parameter, update the &lt;tt&gt;mongod.conf&lt;/tt&gt; files for the Primary and Secondary nodes to include the following and restart the process:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;   wiredTiger:&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      engineConfig:&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;         configString: file_manager=(close_handle_minimum=100,close_idle_time=30,close_scan_interval=30)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;As a reminder, these settings are suitable whenever an Application has a large number of tables (a table is either a collection or an index) which only regularly access a relatively small proportion of them. Applications that consistently access data from across a large set of collections and indexes are likely to see a worse performance with the suggested configuration.&lt;/p&gt;

&lt;p&gt;Thank you,&lt;br/&gt;
Dima&lt;/p&gt;</comment>
                            <comment id="2099824" author="sallgeud" created="Mon, 24 Dec 2018 23:01:42 +0000"  >&lt;p&gt;How would one set those specific WT parameters? Or where?&#160; I assume this works on 3.4?&lt;/p&gt;

&lt;p&gt;I&apos;d love to share how we organize our data and indexes in a way that would be helpful. If you all want, I can set up a web conference and show / discuss our methods.&lt;/p&gt;

&lt;p&gt;We&apos;ve got hundreds of databases. Our method to get around these file limitations, which include time-to-boot (very long), and various problems that seem greatly exacerbated in 3.6 (vs 3.4), is to simply have more physical hardware and/or more services.&#160; So when we have a machine that&apos;s running into file problems, but has plenty of memory and disk, we simply configure another service with a different port and data directory. It allows us to scale within a machine, but it&apos;s a poor scaling method.&lt;/p&gt;

&lt;p&gt;Unfortunately, sharding is nearly pointless in our environment, as &amp;gt; 90% of customers will never really use more than 50GB (we&apos;re very data-light, on purpose).&lt;/p&gt;</comment>
                            <comment id="2086754" author="dmitry.agranat" created="Tue, 11 Dec 2018 11:25:15 +0000"  >&lt;p&gt;Hi Chad,&lt;/p&gt;

&lt;p&gt;I&apos;d like to address your latest comment about the large amount of files.&lt;/p&gt;

&lt;p&gt;Under certain workloads, the number of active files MongoDB uses can impact a system&apos;s performance. Sometimes the bottleneck in these workloads is the eviction server single thread identifying candidate pages for eviction (since we need to walk many trees).&lt;/p&gt;

&lt;p&gt;To mitigate this issues, we can try to speed up the time when we need to close &lt;b&gt;non-active&lt;/b&gt; &lt;tt&gt;dhandles&lt;/tt&gt; by setting this WT parameter:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;file_manager=(close_handle_minimum=100,close_idle_time=30,close_scan_interval=30) &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Metrics explanation:&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;&lt;tt&gt;close_scan_interval&lt;/tt&gt;: interval in seconds at which to check for files that are inactive and close them&lt;/li&gt;
	&lt;li&gt;&lt;tt&gt;close_idle_time&lt;/tt&gt;: the amount of time in seconds a file handle needs to be idle before attempting to close it&lt;/li&gt;
	&lt;li&gt;&lt;tt&gt;close_handle_minimum&lt;/tt&gt;: number of handles open before the file manager will look for handles to close&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;The default to TTL non-active &lt;tt&gt;dhandles&lt;/tt&gt; is every 100000 seconds.&lt;/p&gt;

&lt;p&gt;Please note that these settings are suitable whenever an Application has a large number of tables (a table is either a collection or an index) which only regularly access a relatively small proportion of them. Applications that consistently access data from across a large set of collections and indexes are likely to see a worse performance with the suggested configuration.&lt;/p&gt;

&lt;p&gt;Another mitigation with a large number of tables is trying to avoid unnecessary making a table as active. For example, when you execute &lt;a href=&quot;https://docs.mongodb.com/manual/reference/command/collStats/index.html&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;collStats()&lt;/a&gt; command, you effectively &quot;touch&quot; this collection and all the associated indexes, making all these tables as active. &lt;/p&gt;

&lt;p&gt;In case your Application needs to &quot;touch&quot; &lt;b&gt;many&lt;/b&gt; tables &lt;b&gt;frequently&lt;/b&gt;, effectively making a large number of tables and cursors as active, you might consider horizontal scaling by sharding while leaving individual collections unsharded.&lt;/p&gt;

&lt;p&gt;We&apos;ve deprioritized &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17675&quot; title=&quot;Support file per database for WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17675&quot;&gt;&lt;del&gt;SERVER-17675&lt;/del&gt;&lt;/a&gt; in favor other solutions in the same space that may provide additional benefits to our users. In particular, I&apos;d like to mention wildcard indexes which will be included in MongoDB 4.2. Based on your comment above, I hope that this feature will help your use-case. However, it&apos;d be helpful if we better understood the number of collections, indexes, and the types of queries your cluster needs to satisfy.&lt;/p&gt;

&lt;p&gt;Wildcard indexes, which will be documented under &lt;a href=&quot;https://jira.mongodb.org/browse/DOCS-12272&quot; title=&quot;(PM-1040) Wildcard Indexes&quot; class=&quot;issue-link&quot; data-issue-key=&quot;DOCS-12272&quot;&gt;&lt;del&gt;DOCS-12272&lt;/del&gt;&lt;/a&gt;, will enable users to create and query against an index that stores keys for multiple fields in the collection. We expect this feature to significantly reduce the number of indexes required to support some workloads and improve performance by reducing the number of active data handles. As well as reducing the operational overhead of ensuring indexes satisfy every query and no indexes are superfluous.&lt;/p&gt;

&lt;p&gt;Thank you,&lt;br/&gt;
Dima&lt;/p&gt;</comment>
                            <comment id="2082267" author="sallgeud" created="Thu, 6 Dec 2018 05:04:05 +0000"  >&lt;p&gt;I&apos;m going to say &quot;probably&quot;. We&apos;ve now got several clusters in production. The 3.6 cluster now has substantially less files on it than the 3.4 cluster. It seems like with low file count, we&apos;re fine, but as we get into the 10k-100k range somewhere, we get the badness.&#160; Our research suggests we could cut the number of indexes we have probably in half, which will help a tiny bit, but when you&apos;ve got 400k files in a data directory, cutting it to 200k only seems to help a little.&lt;/p&gt;

&lt;p&gt;I&apos;d love to see a single file for indexes, which used to be identified for 4, but appears to have been indefinitely postponed.&lt;/p&gt;</comment>
                            <comment id="2082000" author="thomas.schubert" created="Wed, 5 Dec 2018 23:46:08 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sallgeud&quot; class=&quot;user-hover&quot; rel=&quot;sallgeud&quot;&gt;sallgeud&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Is this still an issue for you? Have you been able to gather diagnostic.data and logs from 3.4 and 3.6 nodes running against the same workload that shows a regression?&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Kelsey&lt;/p&gt;</comment>
                            <comment id="2002626" author="sallgeud" created="Fri, 14 Sep 2018 00:19:18 +0000"  >&lt;p&gt;Also, if you could privately email me the actual log lines that caused those 2... I&apos;m having difficulty locating the specifc log file we sent over.&lt;/p&gt;</comment>
                            <comment id="2002624" author="sallgeud" created="Fri, 14 Sep 2018 00:17:27 +0000"  >&lt;p&gt;We don&apos;t have a way to downgrade the system on which these were created. I could share logs of a system which has several orders of magnitude more traffic. However, that system has quite a bit more memory, and the fastest enterprise disk in existence (Optane).&#160;&lt;/p&gt;

&lt;p&gt;That said, we had the exact same issue in product in the 2 week period that it ran a mix of 3.4 and 3.6.&#160; Our only way of generating a mix of both is to upgrade 1 server in production, then make it primary. However, that would have a fairly significant impact on our users, so we&apos;d have to figure out how exactly we&apos;d go about it, without causing too much pain.&lt;/p&gt;</comment>
                            <comment id="1993040" author="nick.brewer" created="Tue, 4 Sep 2018 20:42:46 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sallgeud&quot; class=&quot;user-hover&quot; rel=&quot;sallgeud&quot;&gt;sallgeud&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bgentry&quot; class=&quot;user-hover&quot; rel=&quot;bgentry&quot;&gt;bgentry&lt;/a&gt; Thanks for your patience. I&#8217;m curious where you&#8217;re seeing that slow queries occur in approximately 60 second batches - looking at the logs, slow queries do not seem to follow such a pattern. For example, a roughly two minute window that contains multiple slow queries:&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;a id=&quot;195335_thumb&quot; href=&quot;https://jira.mongodb.org/secure/attachment/195335/195335_1-17.png&quot; title=&quot;1-17.png&quot; file-preview-type=&quot;image&quot; file-preview-id=&quot;195335&quot; file-preview-title=&quot;1-17.png&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/thumbnail/195335/_thumb_195335.png&quot; style=&quot;border: 0px solid black&quot; role=&quot;presentation&quot;/&gt;&lt;/a&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;From the information you&#8217;ve provided, it appears that the most pronounced periods of increased slowness are directly correlated with numerous extremely large collection scans. For example, during the period of slowness exhibited above, where both disk read and the number of documents returned increased dramatically, multiple large collection scans were performed:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2018-07-03T20:16:56.473-0500 I COMMAND  [conn365468] command db.collection command: find { find: &quot;A&quot;, filter: { example: { $ne: null } }, projection: { _id: 1 }, $readPreference: { mode: &quot;secondaryPreferred&quot; }, $db: &quot;db&quot; } planSummary: COLLSCAN keysExamined:0 docsExamined:170221 cursorExhausted:1 numYields:1773 nreturned:0 reslen:107 locks:{ Global: { acquireCount: { r: 3548 } }, Database: { acquireCount: { r: 1774 } }, Collection: { acquireCount: { r: 1774 } } } protocol:op_query 52012ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Note that I&#8217;ve sanitized the collection / database names in this query, since this ticket is public. Another example from the same time frame:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2018-07-03T20:17:35.570-0500 I COMMAND  [conn365459] command db.collection command: find { find: &quot;B&quot;, filter: { example: { $ne: null }, example2: { $size: 1 } }, projection: { _id: 1, B: 1, C: 1 }, $readPreference: { mode: &quot;secondaryPreferred&quot; }, $db: &quot;db&quot; } planSummary: COLLSCAN keysExamined:0 docsExamined:97889 cursorExhausted:1 numYields:890 nreturned:0 reslen:106 locks:{ Global: { acquireCount: { r: 1782 } }, Database: { acquireCount: { r: 891 } }, Collection: { acquireCount: { r: 891 } } } protocol:op_query 17133ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;The slow query instances that we&#8217;ve managed to find so far all match up with periods where several collection scans were performed with a large number of documents examined. In order to demonstrate a regression, we would need to see logs and &lt;tt&gt;diagnostic.data&lt;/tt&gt; for a 3.4 primary with the same workload.&lt;/p&gt;

&lt;p&gt;-Nick&lt;/p&gt;</comment>
                            <comment id="1992047" author="sallgeud" created="Tue, 4 Sep 2018 03:46:15 +0000"  >&lt;p&gt;It makes me wonder if the combination of fairly regular deletes across hundreds (out of thousands) of collections could be a major culprit. Was there anything that changed from 3.4.x to 3.6.x either to how files are managed in WT or how TTL deletes are done?&lt;/p&gt;</comment>
                            <comment id="1988658" author="bgentry" created="Wed, 29 Aug 2018 12:35:49 +0000"  >&lt;p&gt;In one of our logs, we noticed that the slow requests were occurring in batches just slightly more than 60 seconds apart.&#160; This weekend, I saw in the docs that the background task that removes expired documents based on TTL indexes runs every 60 seconds.&#160; Could this be what is causing the performance issue?&lt;/p&gt;</comment>
                            <comment id="1970827" author="nick.brewer" created="Wed, 8 Aug 2018 20:17:12 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sallgeud&quot; class=&quot;user-hover&quot; rel=&quot;sallgeud&quot;&gt;sallgeud&lt;/a&gt; I agree that the logging of secondaryPreferred is confusing - though in my testing so far, I haven&apos;t seen it when a &lt;em&gt;different&lt;/em&gt; log level is set on an actual query; for example I see the correct result in the logs when running something like &lt;tt&gt;db.collection.find().readPref(&apos;primaryPreferred&apos;)&lt;/tt&gt;.&#160;I&apos;m still testing this, and it may result in a separate SERVER ticket to get some clarification.&#160;&lt;/p&gt;

&lt;p&gt;The read concern change I mentioned in my previous update is one possible cause for the increased slowness you&apos;re seeing, but I&apos;m taking a closer look at the diagnostic information you provided in your previous ticket, just to narrow down the potential source.&#160;&lt;/p&gt;

&lt;p&gt;-Nick&lt;/p&gt;</comment>
                            <comment id="1969301" author="sallgeud" created="Tue, 7 Aug 2018 16:32:05 +0000"  >&lt;p&gt;If these need to be broken out into separate issues, we can do that. However, a severe performance regression still exists.&#160; And the incorrect logging of the query type still exist, independently.&#160; I wouldn&apos;t care if the logging took a ton of time to fix. However, we really need the performance issues fixed, soon.&lt;/p&gt;</comment>
                            <comment id="1968158" author="sallgeud" created="Mon, 6 Aug 2018 20:06:08 +0000"  >&lt;p&gt;Fair enough. Though, in our replicaSet, it seems strange that a query that is explicitly called as primaryPreferred would ever log as secondaryPreferred... which made us assume it was potentially related.&lt;/p&gt;</comment>
                            <comment id="1968139" author="nick.brewer" created="Mon, 6 Aug 2018 19:53:14 +0000"  >&lt;p&gt;In 3.6 and above, MongoDB logs the slaveOK bit as secondaryPreferred - because of this, you &lt;b&gt;can&lt;/b&gt; in fact see secondaryPreferred in your logs, even in a standalone environment. This change was part of the work outlined here: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-28814&quot; title=&quot;Eliminate ServerSelectionMetadata in favor of just using ReadPreferenceSetting&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-28814&quot;&gt;&lt;del&gt;SERVER-28814&lt;/del&gt;&lt;/a&gt;.  It&apos;s unlikely that these messages are the cause of any increased slowness you&apos;re seeing as a part of your move to 3.6. &lt;/p&gt;

&lt;p&gt;One thing I should note is that 3.6 &lt;a href=&quot;https://docs.mongodb.com/manual/release-notes/3.6/#read-concern&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;makes the &quot;majority&quot; read concern default&lt;/a&gt;. &lt;/p&gt;

&lt;p&gt;-Nick&lt;/p&gt;



</comment>
                            <comment id="1958099" author="sallgeud" created="Fri, 27 Jul 2018 01:36:56 +0000"  >&lt;p&gt;The node from which the logs were sent was a primary system 100% of the time. Our connection (through C# driver), does not make any secondary queries. So it shouldn&apos;t be sending secondaryPreferred. When we see slow query logs on the exact same databases but under 3.4, they do not log secondaryPreferred.&lt;/p&gt;

&lt;p&gt;We have one command line tool that&apos;s run once a week that makes an aggregation query. That&apos;s literally the only place anywhere in any of our code that we would do anything related to secondaryPreferred.&#160;&lt;/p&gt;

&lt;p&gt;Point being, you should basically &lt;b&gt;never&lt;/b&gt; see a secondaryPreferred query out of our system. We&apos;re seeing them logged by the thousands. We&apos;re also happening to notice that we&apos;re seeing a &lt;b&gt;ton&lt;/b&gt; more slow queries on the exact same databases.&#160; &#160;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;We migrated back to 3.4 in production and 100% of our performance issues were immediately resolved.&lt;/p&gt;</comment>
                            <comment id="1957940" author="nick.brewer" created="Thu, 26 Jul 2018 21:04:39 +0000"  >&lt;p&gt; &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sallgeud&quot; class=&quot;user-hover&quot; rel=&quot;sallgeud&quot;&gt;sallgeud&lt;/a&gt; Drivers that perform direct connections to a mongod &lt;a href=&quot;https://github.com/mongodb/specifications/blob/master/source/server-selection/server-selection.rst#the-slaveok-wire-protocol-flag&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;set the slaveOK bit by default&lt;/a&gt;. The server logs the slaveOK bit as &quot;secondaryPreferred&quot; because that&apos;s the read preference that most closely matches the behavior of slaveOK.&lt;/p&gt;

&lt;p&gt;Picking up from the other ticket:&lt;/p&gt;

&lt;blockquote&gt;
&lt;p&gt;After the 3.6 node became primary, you will also see a lot of slow (multi-second) queries with &lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;$readPreference: { mode: &quot;secondaryPreferred&quot; } &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Looking at the logs you provided previously, it appears that both the &quot;before&quot; and &quot;after&quot; primary switch logs contain multiple entries with &lt;tt&gt;secondaryPreferred&lt;/tt&gt;. Just to clarify, these logs are both from the 3.6 node, before and after it is elected as primary, correct? &lt;/p&gt;

&lt;p&gt;Thanks, &lt;br/&gt;
Nick&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="195335" name="1-17.png" size="280039" author="nick.brewer" created="Tue, 4 Sep 2018 20:37:55 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>20.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Thu, 26 Jul 2018 15:15:34 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        5 years, 1 week ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>backlog-server-pm</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            5 years, 1 week ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>bgentry</customfieldvalue>
            <customfieldvalue>sallgeud</customfieldvalue>
            <customfieldvalue>dmitry.agranat@mongodb.com</customfieldvalue>
            <customfieldvalue>kelsey.schubert@mongodb.com</customfieldvalue>
            <customfieldvalue>nick.brewer</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hu3gin:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|htu4sn:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[dmitry.agranat@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hu32rz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>