<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:24:45 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-30710] Several seconds for indexed query - global lock?</title>
                <link>https://jira.mongodb.org/browse/SERVER-30710</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;I find, in the slow query logs, queries that make no sense. An excerpt is below:&lt;/p&gt;


&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-08-17T09:32:03.964+0200 I COMMAND  [conn8550] query XX.statistic query: { date: new Date(1502917200000), monitor: [ &quot;monitor|general&quot; ] } planSummary: IXSCAN { monitor: 1, date: 1 } ntoskip:0 nscanned:1 nscannedObjects:1 keyUpdates:0 writeConflicts:0 numYields:2 nreturned:1 reslen:155 locks:{ Global: { acquireCount: { r: 6 } }, Database: { acquireCount: { r: 3 } }, Collection: { acquireCount: { r: 3 } } } 23606ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-08-17T09:32:03.964+0200 I COMMAND  [conn8519] query XX.trigger query: { token: &quot;live/dynamic_subscription/live&quot;, status: true, retired: false, ref: 17 } planSummary: IXSCAN { token: 1 } ntoskip:0 nscanned:1 nscannedObjects:1 keyUpdates:0 writeConflicts:0 numYields:2 nreturned:1 reslen:13278 locks:{ Global: { acquireCount: { r: 6 } }, Database: { acquireCount: { r: 3 } }, Collection: { acquireCount: { r: 3 } } } 33632ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-08-17T09:32:03.964+0200 I COMMAND  [conn8493] query XX.trigger query: { token: &quot;live/product_recommender/live&quot;, status: true, retired: false, ref: 20 } planSummary: IXSCAN { token: 1 } ntoskip:0 nscanned:1 nscannedObjects:1 keyUpdates:0 writeConflicts:0 numYields:2 nreturned:1 reslen:22196 locks:{ Global: { acquireCount: { r: 6 } }, Database: { acquireCount: { r: 3 } }, Collection: { acquireCount: { r: 3 } } } 29860ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-08-17T09:32:03.970+0200 I COMMAND  [conn8518] query XXX.statistic query: { date: new Date(1502917200000), monitor: [ &quot;monitor|general&quot; ] } planSummary: IXSCAN { monitor: 1, date: 1 } ntoskip:0 nscanned:1 nscannedObjects:1 keyUpdates:0 writeConflicts:0 numYields:2 nreturned:1 reslen:155 locks:{ Global: { acquireCount: { r: 6 } }, Database: { acquireCount: { r: 3 } }, Collection: { acquireCount: { r: 3 } } } 20648ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;20 seconds for a query that hits an index?&lt;/p&gt;

&lt;p&gt;Sometimes I also get queries on an ID:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-08-17T10:00:19.909+0200 I COMMAND  [conn10311] query XX.dlv_cs_similar query: { _id: ObjectId(&apos;59954d0fcaf95bd83e8b51f6&apos;) } planSummary: IDHACK ntoskip:0 nscanned:0 nscannedObjects:0 idhack:1 keyUpdates:0 writeConflicts:0 numYields:0 nreturned:0 reslen:20 locks:{ Global: { acquireCount: { r: 2 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { r: 1 } } } 4701ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;It occurs suddenly, every several minutes. It affects 20-30 queries, than nothing. It&apos;s like, from time to time, there is a global lock that occurs.&lt;/p&gt;

&lt;p&gt;More details about my setup:&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;System: Debian 9&lt;/li&gt;
	&lt;li&gt;Mongo version: 3.0.15&lt;/li&gt;
	&lt;li&gt;2 SSD disks in raid0&lt;/li&gt;
&lt;/ul&gt;


&lt;ul&gt;
	&lt;li&gt;Mongo RAM usage: 13.4 GB / 128 G&lt;/li&gt;
	&lt;li&gt;CPU @20%&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Mongostat does not give anything suspicious:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;12  3781    333     96     174   525|0     1.6   18.5       0 16.7G 15.5G   0|1  21|9  641k    12m  155 rplset04  PRI 09:54:14&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    12  4344    380    105     188   295|0     1.6   18.5       0 16.7G 15.5G   0|0 23|12  708k    11m  137 rplset04  PRI 09:54:15&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    10  3221    319    133     163   241|0     1.7   18.5       0 16.7G 15.5G   0|0  23|7  543k    14m  128 rplset04  PRI 09:54:16&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    18  2915    319    114     172    90|0     1.7   18.5       0 16.7G 15.5G   0|0 22|10  511k    17m  128 rplset04  PRI 09:54:17&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;


&lt;p&gt;One primary, one replica, one voter.&lt;/p&gt;

&lt;p&gt;For multitenancy, I am using one DB per client (600 databases). They take 185 GB on the disk.&lt;/p&gt;</description>
                <environment></environment>
        <key id="417473">SERVER-30710</key>
            <summary>Several seconds for indexed query - global lock?</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="4">Incomplete</resolution>
                                        <assignee username="dmitry.agranat@mongodb.com">Dmitry Agranat</assignee>
                                    <reporter username="vladzloteanu">Vlad Zloteanu</reporter>
                        <labels>
                    </labels>
                <created>Thu, 17 Aug 2017 08:05:03 +0000</created>
                <updated>Mon, 9 Oct 2017 17:01:51 +0000</updated>
                            <resolved>Thu, 7 Sep 2017 13:08:48 +0000</resolved>
                                                                    <component>WiredTiger</component>
                                        <votes>0</votes>
                                    <watches>7</watches>
                                                                                                                <comments>
                            <comment id="1666686" author="dmitry.agranat" created="Thu, 7 Sep 2017 10:51:24 +0000"  >&lt;p&gt;Hi Vlad, &lt;/p&gt;

&lt;p&gt;Thank you for providing the requested information.&lt;/p&gt;

&lt;p&gt;I have reviewed the uploaded data. During the reported time frame, the operation rate is between ~18K and 24K and for 99.9% of the time, the operation response time is ~240ms. There are 8 occurrences where we can see a few queries are taking ~20sec. Unfortunately, the data you&apos;ve uploaded did not suffice the cause of these occurrences. This does not mean that there is no problem, it just means that currently, in 3.0.15 the diagnostic data does not expose the potential cause.&lt;/p&gt;

&lt;p&gt;As mentioned earlier by Thomas, I recommend upgrading the cluster to one of our latest versions, preferably 3.4.7. Apart from many significant improvements since 3.0.x versions, we have also significantly improved our diagnostics capabilities, which would help us to better understand the cause of these occurrences.&lt;/p&gt;

&lt;p&gt;If after upgrading your cluster you still see this issue reoccurs, please open a new case and attach an archive of the diagnostic.data directory in your $dbpath and the complete mongod log files.&lt;/p&gt;

&lt;p&gt;Thank you,&lt;br/&gt;
Dima&lt;/p&gt;</comment>
                            <comment id="1659321" author="vladzloteanu" created="Tue, 29 Aug 2017 08:27:03 +0000"  >&lt;p&gt;Hello @Thomas Schubert,&lt;/p&gt;

&lt;p&gt;Did you get the chance to look into the logs I uploaded?&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Vlad&lt;/p&gt;</comment>
                            <comment id="1653735" author="vladzloteanu" created="Tue, 22 Aug 2017 13:28:42 +0000"  >&lt;p&gt;Thank you for your reply, Thomas. &lt;br/&gt;
I have attached the requested logs for 2 servers: &lt;/p&gt;
&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;a single instance mongo, having 1-5k updates per second;&lt;/li&gt;
	&lt;li&gt;the primary from a replicaset, with 2-5 hundreds updates per second&lt;/li&gt;
&lt;/ul&gt;
</comment>
                            <comment id="1653438" author="thomas.schubert" created="Tue, 22 Aug 2017 01:13:40 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=vladzloteanu&quot; class=&quot;user-hover&quot; rel=&quot;vladzloteanu&quot;&gt;vladzloteanu&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Sorry, I missed that you were running MongoDB 3.0, which does not capture diagnostic.data. As Ram&#243;n mentioned in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-30707&quot; title=&quot;server crash: invariant() failure &quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-30707&quot;&gt;&lt;del&gt;SERVER-30707&lt;/del&gt;&lt;/a&gt;, there have been significant improvements to WiredTiger since the release of MongoDB 3.0, and, after reviewing your logs, I believe that it is likely that upgrading will resolve this issue as well. To confirm my understanding of this issue, I&apos;d like some additional information.&lt;/p&gt;

&lt;p&gt;Would you please run the following script for an hour?&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;delay=1&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo --eval &quot;while(true) {print(JSON.stringify(db.serverStatus({tcmalloc:true}))); sleep(1000*${delay:?})}&quot; &amp;gt;ss.log &amp;amp;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;iostat -k -t -x ${delay:?} &amp;gt;iostat.log &amp;amp;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;Afterwards, please provide both generated files, as well as the mongod.log files covering the time period.&lt;/p&gt;

&lt;p&gt;Thank you,&lt;br/&gt;
Thomas&lt;/p&gt;</comment>
                            <comment id="1650867" author="vladzloteanu" created="Thu, 17 Aug 2017 20:03:57 +0000"  >&lt;p&gt;I have uploaded the logs.&lt;br/&gt;
Please disregard everything that happened before 2017-08-17T19:00:00, as some mongorestores were running.&lt;/p&gt;

&lt;p&gt;Here is an example of a line:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-08-17T21:19:49.837+0200 I COMMAND  [conn38102] query db_a***o.buffer query: { _id: ObjectId(&apos;5995ec49d5bc6d9b2c8b4dfe&apos;) } planSummary: IDHACK ntoskip:0 nscanned:0 nscannedObjects:0 idhack:1 keyUpdates:0 writeConflicts:0 numYields:0 nreturned:0 reslen:20 locks:{ Global: { acquireCount: { r: 2 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { r: 1 } } } 12321ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;


&lt;p&gt;Here&apos;s another one:&lt;/p&gt;


&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2017-08-17T21:24:10.617+0200 I COMMAND  [conn39247] query db_c***o.push_dlv_item query: { trigger.keyword: &quot;cart_saver&quot;, receiver.osuserid: &quot;null&quot;, delivery.created_at: { $gte: new Date(1502393033000) } } planSummary: IXSCAN { trigger.keyword: 1, receiver.osuserid: 1, delivery.created_at: 1 } ntoskip:0 nscanned:0 nscannedObjects:0 keyUpdates:0 writeConflicts:0 numYields:0 nreturned:0 reslen:20 locks:{ Global: { acquireCount: { r: 2 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { r: 1 } } } 16736ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;I can also give you access to the datadog monitoring account.&lt;/p&gt;

&lt;p&gt;Please let me know if you need anything else.&lt;/p&gt;</comment>
                            <comment id="1650845" author="vladzloteanu" created="Thu, 17 Aug 2017 19:42:38 +0000"  >&lt;p&gt;Hello Thomas,&lt;br/&gt;
I have no diagnostic.data folder. My current mongo version is 3.0.15.&lt;br/&gt;
Should I have such a folder?&lt;/p&gt;</comment>
                            <comment id="1650521" author="thomas.schubert" created="Thu, 17 Aug 2017 14:52:35 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=vladzloteanu&quot; class=&quot;user-hover&quot; rel=&quot;vladzloteanu&quot;&gt;vladzloteanu&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;Thanks for the report. So we can investigate, would you please provide an archive of the &lt;tt&gt;diagnostic.data&lt;/tt&gt; directory in your &lt;tt&gt;$dbpath&lt;/tt&gt; and the complete mongod log files for the affected node?&lt;/p&gt;

&lt;p&gt;I&apos;ve created a &lt;a href=&quot;https://10gen-httpsupload.s3.amazonaws.com/upload_forms/98cb75ed-ad97-4495-b448-f0ae9aba0070.html&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;secure upload portal&lt;/a&gt; for you to use. Files uploaded to this portal are only visible to MongoDB employees investigating this issue and are routinely deleted after some time.&lt;/p&gt;

&lt;p&gt;Thank you,&lt;br/&gt;
Thomas &lt;/p&gt;</comment>
                            <comment id="1650392" author="vladzloteanu" created="Thu, 17 Aug 2017 12:49:43 +0000"  >&lt;p&gt;Also, as there are 600+ databases, the mongodb process has around 62k open FDs.&lt;/p&gt;</comment>
                            <comment id="1650278" author="vladzloteanu" created="Thu, 17 Aug 2017 08:13:41 +0000"  >&lt;p&gt;I am available for any other information necessary. &lt;br/&gt;
I can provide the newrelic and datadog monitoring.&lt;/p&gt;

&lt;p&gt;They are the same servers as on &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-30707&quot; title=&quot;server crash: invariant() failure &quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-30707&quot;&gt;&lt;del&gt;SERVER-30707&lt;/del&gt;&lt;/a&gt;.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="417316">SERVER-30707</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Thu, 17 Aug 2017 14:52:35 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        6 years, 22 weeks, 6 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>backlog-server-pm</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            6 years, 22 weeks, 6 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>dmitry.agranat@mongodb.com</customfieldvalue>
            <customfieldvalue>kelsey.schubert@mongodb.com</customfieldvalue>
            <customfieldvalue>vladzloteanu</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htd5v3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|ht4y2v:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[kelsey.schubert@mongodb.com]]></customfieldvalue>
        <customfieldvalue><![CDATA[dmitry.agranat@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htcrxz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>