<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:49:28 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-18988] Terrible Query Performance</title>
                <link>https://jira.mongodb.org/browse/SERVER-18988</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;We are facing to a terrible query performance issue. There are only 5 million rows in the table. And the we add in index on username, the query we run are always against the username. Both nscaned and nscannedObjects are 1, however it will take more than 100 ms.&lt;/p&gt;</description>
                <environment></environment>
        <key id="210846">SERVER-18988</key>
            <summary>Terrible Query Performance</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="5">Cannot Reproduce</resolution>
                                        <assignee username="bruce.lucas@mongodb.com">Bruce Lucas</assignee>
                                    <reporter username="cainiaoiswo">JinHui</reporter>
                        <labels>
                    </labels>
                <created>Tue, 16 Jun 2015 03:27:38 +0000</created>
                <updated>Thu, 10 Mar 2016 22:38:18 +0000</updated>
                            <resolved>Mon, 7 Mar 2016 13:16:17 +0000</resolved>
                                    <version>3.0.3</version>
                                                    <component>Performance</component>
                    <component>WiredTiger</component>
                                        <votes>0</votes>
                                    <watches>18</watches>
                                                                                                                <comments>
                            <comment id="1198584" author="carl.dong@windfindtech.com" created="Thu, 10 Mar 2016 01:33:48 +0000"  >&lt;p&gt;Sure , here is the log for 3 days .&lt;/p&gt;</comment>
                            <comment id="1197536" author="bruce.lucas@10gen.com" created="Wed, 9 Mar 2016 13:24:51 +0000"  >&lt;p&gt;Hi Carl,&lt;/p&gt;

&lt;p&gt;Would you be able to also upload the mongod log showing the slow queries so that we can correlate them with events in the diagnostic data.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Bruce&lt;/p&gt;</comment>
                            <comment id="1197310" author="carl.dong@windfindtech.com" created="Wed, 9 Mar 2016 07:29:25 +0000"  >&lt;p&gt;Hi Ramon,&lt;/p&gt;

&lt;p&gt;Tested Version 3.2.3 , this time I observe the issue happens everytime 3-4 seconds after database flush action, modify the flush interval from 60 to 20 seconds , the issue queries reduce but still exist.&lt;br/&gt;
Upload diagnostic files as attachement , please let us know if we could tune some parameters(maybe hidden ones).&lt;/p&gt;

&lt;p&gt;Thanks ,&lt;/p&gt;

&lt;p&gt;Carl Dong&lt;/p&gt;</comment>
                            <comment id="1194831" author="ramon.fernandez" created="Mon, 7 Mar 2016 13:16:09 +0000"  >&lt;p&gt;Unfortunately I have no more information to provide at this point, sorry about that. I was not able to find the root cause with the provided data, nor was I able to reproduce this behavior on my end. Without any additional information that shows evidence of a bug in the server I&apos;m afraid I have to close this ticket for the time being.&lt;/p&gt;

&lt;p&gt;As &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-18988?focusedCommentId=1160196&amp;amp;page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-1160196&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;I mentioned above&lt;/a&gt;, I&apos;d recommend testing MongoDB 3.2.3. If the behavior persists please post in this ticket and I&apos;ll be happy to reopen it.&lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="1190080" author="cainiaoiswo" created="Wed, 2 Mar 2016 09:10:12 +0000"  >&lt;p&gt;As you can see from Carl&apos;s screenshot, all of the slow query happened in batch. And most of the slow query are update or insert command, there is now key update for these query. Any thoughts?&lt;/p&gt;</comment>
                            <comment id="1160571" author="carl.dong@windfindtech.com" created="Tue, 2 Feb 2016 05:39:59 +0000"  >&lt;p&gt;In fact , the issue frequently happens , as you see in the ops manager profiler page.  &lt;/p&gt;</comment>
                            <comment id="1160196" author="ramon.fernandez" created="Mon, 1 Feb 2016 21:34:11 +0000"  >&lt;p&gt;Carl, since I haven&apos;t found a clear root cause I can&apos;t say for sure whether upgrading will help. Note however that MongoDB 3.2 makes WiredTiger the default storage engine, and a lot of work was put into making things faster and more stable &amp;#8211; so if this turns out to be an unfortunate interaction with the storage engine then 3.2.1 (our latest stable release right now) should definitely help.&lt;/p&gt;

&lt;p&gt;MongoDB 3.2 also makes it easier to collect diagnostics data, as such data is recorded in the background and there&apos;s no need to wait until a given problem happens again.&lt;/p&gt;</comment>
                            <comment id="1156191" author="carl.dong@windfindtech.com" created="Thu, 28 Jan 2016 02:56:42 +0000"  >&lt;p&gt;Hi Ramon,&lt;/p&gt;

&lt;p&gt;We are running on Mongodb-enterprise 3.0.7 , but issue still exists , even worse . .&lt;/p&gt;

&lt;p&gt;If diagnostic data would help in 3.2.0 version , we will plan a upgrade for that .&lt;/p&gt;

&lt;p&gt;Thanks ,&lt;/p&gt;

&lt;p&gt;Carl Dong&lt;/p&gt;</comment>
                            <comment id="1155764" author="ramon.fernandez" created="Wed, 27 Jan 2016 20:20:44 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=cainiaoiswo&quot; class=&quot;user-hover&quot; rel=&quot;cainiaoiswo&quot;&gt;cainiaoiswo&lt;/a&gt;, apologies for the long delay in getting back to you. We were not able to identify any concluding evidence of what may be causing the slow queries reported. The data you sent us didn&apos;t allow us to identify what resource may be being temporarily held and slowing the queries down.&lt;/p&gt;

&lt;p&gt;I&apos;ll take another look at the uploaded data to see if we missed something. Am I right to assume you&apos;re still experiencing this issue? Or has a recent upgrade helped speed queries up?&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="1066629" author="cainiaoiswo" created="Wed, 21 Oct 2015 09:03:28 +0000"  >&lt;p&gt;Hi Ramon, is there any good news for me?&lt;/p&gt;</comment>
                            <comment id="1025515" author="ramon.fernandez" created="Wed, 9 Sep 2015 18:44:36 +0000"  >&lt;p&gt;Apologies for the radio silence &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=cainiaoiswo&quot; class=&quot;user-hover&quot; rel=&quot;cainiaoiswo&quot;&gt;cainiaoiswo&lt;/a&gt; - yes, this issue is till open and is being investigated. The information collected so far is not conclusive as of yet; we&apos;ll post updates to this ticket as they happen.&lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="1004006" author="cainiaoiswo" created="Thu, 13 Aug 2015 02:24:21 +0000"  >&lt;p&gt;Please let me know if you are still working on this issue.&lt;/p&gt;</comment>
                            <comment id="993522" author="cainiaoiswo" created="Fri, 31 Jul 2015 03:50:25 +0000"  >&lt;p&gt;Hi Bruce, We&apos;ve upgraded our DB to 3.0.5 but the issue still exists. By the way,  you&apos;d better use multiple threads to reproduce the issue, The number of the slow query during peak time is hundreds time than other time.&lt;/p&gt;</comment>
                            <comment id="968612" author="cainiaoiswo" created="Fri, 17 Jul 2015 07:42:30 +0000"  >&lt;p&gt;Hi Bruce, thanks for your update. I&apos;ve attached more information in the thread.&lt;br/&gt;
For question 1: Most of the slow query are write. Sometime we can also find some slow read query but just a little.&lt;br/&gt;
For question 2: I&apos;ve turned off the profiler.&lt;/p&gt;</comment>
                            <comment id="968088" author="bruce.lucas@10gen.com" created="Thu, 16 Jul 2015 18:28:51 +0000"  >&lt;p&gt;Hi JinHui,&lt;/p&gt;

&lt;p&gt;Thanks for providing that information, and sorry for the delay in responding.&lt;/p&gt;

&lt;p&gt;That data shows one brief incident, at 20:20:20, where a dozen write (insert and update operations) all completed within about 1ms of each other, the longest of them having taken 640ms, suggesting that they had been held up accessing some common resource and then all completed quickly when that resource became available. It affects a couple of different collections, and as you have noted the issue often affects more than one database at the same time, suggesting that the common resource is in some lower layer of the system, such as the storage engine or storage itself.&lt;/p&gt;

&lt;p&gt;In the data we do find some correlation with that one incident and some internal mongod counters. (I attached a graph that we are using for internal discussions about the issue showing that correlation.)  However, as it is only one brief incident is too little for drawing conclusions. I hate to have to ask you for more information, but would you be able run the same data collection (serverStatus, iostat, and mongod log) for a longer period of time, maybe a few hours or a day, suffcient to cover a few incidents, so that we can see if they all follow the same pattern?&lt;/p&gt;

&lt;p&gt;Also let me ask a couple questions:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;are the slow operations you are concerned with always writes (inserts or updates)?&lt;/li&gt;
	&lt;li&gt;can you confirm that you have turned profiling off? I just want to rule out the possibility that it is having an impact.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Thanks,&lt;br/&gt;
Bruce&lt;/p&gt;</comment>
                            <comment id="966433" author="cainiaoiswo" created="Wed, 15 Jul 2015 08:21:13 +0000"  >&lt;p&gt;The issue still exists.&lt;br/&gt;
/* 5 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 830,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.188Z&quot;)
}

&lt;p&gt;/* 6 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 560,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.188Z&quot;)
}

&lt;p&gt;/* 7 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 534,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.188Z&quot;)
}

&lt;p&gt;/* 8 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 281,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.188Z&quot;)
}

&lt;p&gt;/* 9 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 694,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.188Z&quot;)
}

&lt;p&gt;/* 10 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 730,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.188Z&quot;)
}

&lt;p&gt;/* 11 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 720,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.187Z&quot;)
}

&lt;p&gt;/* 12 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 428,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.187Z&quot;)
}

&lt;p&gt;/* 13 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 451,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.187Z&quot;)
}

&lt;p&gt;/* 14 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 201,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.187Z&quot;)
}

&lt;p&gt;/* 15 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 216,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.187Z&quot;)
}

&lt;p&gt;/* 16 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 898,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.187Z&quot;)
}

&lt;p&gt;/* 17 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 482,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.186Z&quot;)
}

&lt;p&gt;/* 18 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 919,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.186Z&quot;)
}

&lt;p&gt;/* 19 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 464,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.186Z&quot;)
}

&lt;p&gt;/* 20 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 509,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.186Z&quot;)
}

&lt;p&gt;/* 21 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 234,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.186Z&quot;)
}

&lt;p&gt;/* 22 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 823,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.186Z&quot;)
}

&lt;p&gt;/* 23 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 264,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.186Z&quot;)
}

&lt;p&gt;/* 24 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 766,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.186Z&quot;)
}

&lt;p&gt;/* 25 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 315,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.186Z&quot;)
}

&lt;p&gt;/* 26 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 458,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.186Z&quot;)
}

&lt;p&gt;/* 27 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 677,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.185Z&quot;)
}

&lt;p&gt;/* 28 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 274,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.185Z&quot;)
}

&lt;p&gt;/* 29 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 444,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.185Z&quot;)
}

&lt;p&gt;/* 30 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 660,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.185Z&quot;)
}

&lt;p&gt;/* 31 */&lt;/p&gt;
{
    &quot;op&quot; : &quot;insert&quot;,
    &quot;millis&quot; : 739,
    &quot;ts&quot; : ISODate(&quot;2015-07-15T07:53:44.185Z&quot;)
}</comment>
                            <comment id="957960" author="cainiaoiswo" created="Sun, 5 Jul 2015 11:19:06 +0000"  >&lt;p&gt;Still can&apos;t reproduce the issue?&lt;/p&gt;</comment>
                            <comment id="954132" author="cainiaoiswo" created="Tue, 30 Jun 2015 02:17:43 +0000"  >&lt;p&gt;Thanks for your updates. Here are some other information may be helpful. We have two database in this server. One is for normal business request, which contains the table DimUser we mention in this thread. The other one is for log request, which records about 200 rows/second. And we find that the slow queries always occur in two databases at the same time.&lt;/p&gt;</comment>
                            <comment id="952942" author="ramon.fernandez" created="Mon, 29 Jun 2015 12:09:13 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=cainiaoiswo&quot; class=&quot;user-hover&quot; rel=&quot;cainiaoiswo&quot;&gt;cainiaoiswo&lt;/a&gt;, we&apos;re still investigating this issue. The last batch of information you sent does show slow queries in the range of hundreds of milliseconds, but haven&apos;t been able to find the cause; I&apos;ve also tried reproducing this on my end, unsuccessfully. I&apos;ll let you know if I need any further information &amp;#8211; thanks for your patience and understanding.&lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="952816" author="cainiaoiswo" created="Mon, 29 Jun 2015 06:30:50 +0000"  >&lt;p&gt;Do you  need any other Log or information from my side?&lt;/p&gt;</comment>
                            <comment id="948842" author="cainiaoiswo" created="Wed, 24 Jun 2015 01:45:14 +0000"  >&lt;p&gt;Hi Ramon, here is the log we caught last night.&lt;/p&gt;</comment>
                            <comment id="947390" author="ramon.fernandez" created="Mon, 22 Jun 2015 20:36:31 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=cainiaoiswo&quot; class=&quot;user-hover&quot; rel=&quot;cainiaoiswo&quot;&gt;cainiaoiswo&lt;/a&gt;, thanks for uploading the requested information. While I do see the long queries in the logs I haven&apos;t been able to find the &quot;smoking gun&quot;, so we&apos;ll need more information to track down the root cause of the issue. I&apos;d like to ask you to run the following commands on the node you&apos;re seeing the slow queries (I presume is your primary):&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo --eval &quot;while(true) {print(JSON.stringify(db.serverStatus({tcmalloc:1}))); sleep(200)}&quot; &amp;gt; ss.log &amp;amp;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;iostat -k -t -x 1 &amp;gt; iostat.log &amp;amp;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;These will monitor your &lt;tt&gt;mongod&lt;/tt&gt; instance and log information to the files &lt;tt&gt;ss.log&lt;/tt&gt; and &lt;tt&gt;iostat.log&lt;/tt&gt; for as long as they run. The hope is to record system information at the same time you&apos;re experiencing slow queries, so please continue to run these commands until you see some slow queries, after which you can interrupt the monitoring commands and upload the &lt;tt&gt;ss.log&lt;/tt&gt; and &lt;tt&gt;iostat.log&lt;/tt&gt; files so we can analyze them.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;
</comment>
                            <comment id="944127" author="cainiaoiswo" created="Thu, 18 Jun 2015 14:23:33 +0000"  >&lt;p&gt;Is there any update for my issue?&lt;/p&gt;</comment>
                            <comment id="943154" author="cainiaoiswo" created="Wed, 17 Jun 2015 17:00:01 +0000"  >&lt;p&gt;Hi Ramon, here is the Level 5 log. And the png file seems like a slow query plan. As I found the query plan was decide at 2015-06-18T00:34:01.000 and it keep running for 285ms.&lt;/p&gt;</comment>
                            <comment id="943078" author="cainiaoiswo" created="Wed, 17 Jun 2015 16:03:48 +0000"  >&lt;p&gt;Here is the server log. As you can see in the file, since line 1000, lots of query cause more then thousand of MS.&lt;/p&gt;</comment>
                            <comment id="943047" author="cainiaoiswo" created="Wed, 17 Jun 2015 15:43:58 +0000"  >&lt;p&gt;The problem is going worse. Lots of query take more than 1000 ms. Seems our production environment is going crush.&lt;/p&gt;</comment>
                            <comment id="942857" author="cainiaoiswo" created="Wed, 17 Jun 2015 13:13:34 +0000"  >&lt;p&gt;Hi Ramon, here is some log info you need.&lt;/p&gt;</comment>
                            <comment id="942847" author="cainiaoiswo" created="Wed, 17 Jun 2015 12:58:40 +0000"  >&lt;p&gt;Most of the time, the query time is OK, which only take about several ms.&lt;/p&gt;</comment>
                            <comment id="942820" author="ramon.fernandez" created="Wed, 17 Jun 2015 12:27:06 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=cainiaoiswo&quot; class=&quot;user-hover&quot; rel=&quot;cainiaoiswo&quot;&gt;cainiaoiswo&lt;/a&gt;, in addition to Dan&apos;s question we&apos;ll need some more information to troubleshoot this issue.&lt;/p&gt;

&lt;ol&gt;
	&lt;li&gt;Would it be possible for you to run the following commands and send us the resulting logs?
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;// Increase log level, disable profiler but log all queries&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;use admin&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;var old_log = db.setLogLevel(2)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;var old_profiler = db.setProfilingLevel(0,0)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;// Run the slow query&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;use DimUser&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&amp;lt;the slow username query originally reported&amp;gt;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;// restore log and profiler values&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;use admin&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;db.runCommand({setParameter:1, logComponentVerbosity:old_log.was})&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;db.setProfilingLevel(old_profiler.was, old_profiler.slowms)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;Please note that this will temporarily disable the profiler altogether.
&lt;br class=&quot;atl-forced-newline&quot; /&gt; &lt;br class=&quot;atl-forced-newline&quot; /&gt;&lt;/p&gt;&lt;/li&gt;
	&lt;li&gt;The output of the following commands may be useful as well:
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;db.stats()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;db.&amp;lt;collection&amp;gt;.stats()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;&lt;br class=&quot;atl-forced-newline&quot; /&gt; &lt;/p&gt;&lt;/li&gt;
	&lt;li&gt;The startup logs for this server&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;Thanks,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="942775" author="dan@10gen.com" created="Wed, 17 Jun 2015 11:45:42 +0000"  >&lt;p&gt;Even without knowing what else is happening on this server, that is an unusually long time for the query and document size.  If you run the same query repeatedly, does the time to execute stay the same?&lt;/p&gt;</comment>
                            <comment id="942761" author="cainiaoiswo" created="Wed, 17 Jun 2015 11:14:51 +0000"  >&lt;p&gt;avgObjSize is 256 Byte.&lt;/p&gt;</comment>
                            <comment id="942731" author="dan@10gen.com" created="Wed, 17 Jun 2015 10:15:39 +0000"  >&lt;p&gt;Can you tell me how large the document you are querying for is?&lt;/p&gt;</comment>
                            <comment id="942612" author="cainiaoiswo" created="Wed, 17 Jun 2015 05:04:56 +0000"  >&lt;p&gt;Our platform is CentOS6.5; MongoDB 3.0.3 WiredTiger&lt;br/&gt;
CPU:2*2670v2 2.5GHz&lt;br/&gt;
Memory: 64GB&lt;br/&gt;
HardDrive: 2TB 7.2K RPM&lt;/p&gt;

&lt;p&gt;Please let me know if you need any other information.&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="78273" name="DimUserStatus.txt" size="7709" author="cainiaoiswo" created="Wed, 17 Jun 2015 13:13:34 +0000"/>
                            <attachment id="79105" name="MongoSlowQueryLog.zip" size="447254" author="cainiaoiswo" created="Wed, 24 Jun 2015 01:45:14 +0000"/>
                            <attachment id="107242" name="OpsManager_ss.png" size="59700" author="carl.dong@windfindtech.com" created="Tue, 2 Feb 2016 05:39:59 +0000"/>
                            <attachment id="78345" name="Slow UserName.png" size="29658" author="cainiaoiswo" created="Wed, 17 Jun 2015 17:00:01 +0000"/>
                            <attachment id="112866" name="dblog.tar" size="1312174" author="carl.dong@windfindtech.com" created="Thu, 10 Mar 2016 01:33:48 +0000"/>
                            <attachment id="78272" name="dbstats.txt" size="318" author="cainiaoiswo" created="Wed, 17 Jun 2015 13:13:34 +0000"/>
                            <attachment id="112673" name="dignostic.tar" size="74438651" author="carl.dong@windfindtech.com" created="Wed, 9 Mar 2016 07:51:30 +0000"/>
                            <attachment id="78340" name="iSH_DB_Log.zip" size="960696" author="cainiaoiswo" created="Wed, 17 Jun 2015 16:03:48 +0000"/>
                            <attachment id="78344" name="iSH_DB_Log_L5.zip" size="1600387" author="cainiaoiswo" created="Wed, 17 Jun 2015 17:00:01 +0000"/>
                            <attachment id="82157" name="incident.png" size="272871" author="bruce.lucas@mongodb.com" created="Thu, 16 Jul 2015 18:27:32 +0000"/>
                            <attachment id="78042" name="profiler.txt" size="3573" author="cainiaoiswo" created="Tue, 16 Jun 2015 03:27:38 +0000"/>
                            <attachment id="78904" name="queries.png" size="84683" author="ramon.fernandez@mongodb.com" created="Mon, 22 Jun 2015 18:48:00 +0000"/>
                            <attachment id="78271" name="startuplog.txt" size="3119" author="cainiaoiswo" created="Wed, 17 Jun 2015 13:13:34 +0000"/>
                            <attachment id="82234" name="status.gz.bz2" size="105906244" author="cainiaoiswo" created="Fri, 17 Jul 2015 07:38:24 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>33.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 17 Jun 2015 10:15:39 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        7 years, 49 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            7 years, 49 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>carl.dong@windfindtech.com</customfieldvalue>
            <customfieldvalue>dan@mongodb.com</customfieldvalue>
            <customfieldvalue>cainiaoiswo</customfieldvalue>
            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrl34f:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hsduj3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[bruce.lucas@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hsfxnz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>