<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 05:20:55 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-49816] mongod server crashed while mongodump was running</title>
                <link>https://jira.mongodb.org/browse/SERVER-49816</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;We are running a replicaSet consisting of 3 bare metal servers. The primary went down while mongodump was executing (daily backup).&lt;br/&gt;
The mongodump command was as follows whereas $HOSTNAME stands for the FQDN of the Primary: &lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongodump -h $HOSTNAME --port $PORT -u${USER} -p${PASS} --authenticationDatabase=$AUTHDATABASE -o $BACKUPDIR &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;We have saved mongod logs and diagnostic data from all 3 replSet members for your investigation.&lt;br/&gt;
Where can we upload them?&lt;/p&gt;

&lt;p&gt;Here are the last lines of the mongod log, which seems not very helpful - at least for us:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2020-07-22T18:53:55.193+0200 I COMMAND  [conn56968] command ipc-catalog.productOfferInfo command: find { find: &quot;productOfferInfo&quot;, filter: { _i&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;d: { _id: 3127816, locale: &quot;de_DE&quot; } }, limit: 1, singleBatch: true, $db: &quot;ipc-catalog&quot;, $clusterTime: { clusterTime: Timestamp(1595436345, 1),&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; signature: { hash: BinData(0, C48B0FBAAB6D9151CD788358E80382121C11A9E5), keyId: 6849703745916239873 } }, lsid: { id: UUID(&quot;629864ef-f24d-4147-&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;89dd-a6c24254de42&quot;) }, $readPreference: { mode: &quot;nearest&quot; } } planSummary: IDHACK numYields:0 ok:0 errMsg:&quot;Executor error during find command :&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;: caused by :: operation was interrupted&quot; errName:InterruptedDueToReplStateChange errCode:11602 reslen:308 locks:{ Global: { acquireCount: { r:&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; 1 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { r: 1 } } } protocol:op_msg 460023ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2020-07-22T18:53:55.193+0200 I REPL     [replexec-1968] Member mongo-india01-02.db00.pro06.eu.idealo.com:27017 is now in state RS_DOWN&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2020-07-22T22:06:29.715+0200 I CONTROL  [main] ***** SERVER RESTARTED *****&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Both mongod and mongodump are of version 4.0.13.&lt;br/&gt;
Each server is running on Linux/Stretch and has 56 CPUs, 384 GB RAM, 5.8 TB SSD in Raid 10.&lt;/p&gt;</description>
                <environment></environment>
        <key id="1417521">SERVER-49816</key>
            <summary>mongod server crashed while mongodump was running</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13202">Works as Designed</resolution>
                                        <assignee username="dmitry.agranat@mongodb.com">Dmitry Agranat</assignee>
                                    <reporter username="kay.agahd@idealo.de">Kay Agahd</reporter>
                        <labels>
                    </labels>
                <created>Wed, 22 Jul 2020 21:05:53 +0000</created>
                <updated>Fri, 27 Oct 2023 13:52:46 +0000</updated>
                            <resolved>Tue, 28 Jul 2020 04:53:12 +0000</resolved>
                                    <version>4.0.13</version>
                                                    <component>Stability</component>
                                        <votes>0</votes>
                                    <watches>6</watches>
                                                                                                                <comments>
                            <comment id="3318011" author="kay.agahd@idealo.de" created="Mon, 3 Aug 2020 16:52:36 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=dmitry.agranat&quot; class=&quot;user-hover&quot; rel=&quot;dmitry.agranat&quot;&gt;dmitry.agranat&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;stating &quot;works as designed&quot; is quite surprising when mongodump impacts performance so badly that the server is marked as DEAD by itself during the dump. &lt;br/&gt;
But if mongodb.org is not willing to fix and not even to investigate it (because you probably want to sell your expensive backup solution), I don&apos;t see a reason to insist any longer.&lt;/p&gt;</comment>
                            <comment id="3317288" author="dmitry.agranat" created="Mon, 3 Aug 2020 11:04:56 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=kay.agahd%40idealo.de&quot; class=&quot;user-hover&quot; rel=&quot;kay.agahd@idealo.de&quot;&gt;kay.agahd@idealo.de&lt;/a&gt;, this ticket is in status &quot;Closed&quot; as we&apos;ve determined the incident was related to your OS/HW issues. As for the latest issue, &lt;tt&gt;mongodump&lt;/tt&gt; can impact the performance of your running database and this is expected. As mentioned earlier, there are other backup methods (including different approaches with mongodump) which you might find more suitable for your cluster.&lt;/p&gt;

&lt;p&gt;The SERVER project is for bugs and feature suggestions for the MongoDB server. If you need further assistance troubleshooting, I encourage you to ask our community by posting on the &lt;a href=&quot;http://community.mongodb.com&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;MongoDB Community Forums&lt;/a&gt; or on &lt;a href=&quot;https://stackoverflow.com/questions/tagged/mongodb&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;Stack Overflow with the &lt;tt&gt;mongodb&lt;/tt&gt; tag&lt;/a&gt;.&lt;/p&gt;</comment>
                            <comment id="3317280" author="kay.agahd@idealo.de" created="Mon, 3 Aug 2020 10:27:13 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=dmitry.agranat&quot; class=&quot;user-hover&quot; rel=&quot;dmitry.agranat&quot;&gt;dmitry.agranat&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;we had the same issue on Sunday, 2020-08-02. We limited mongodump by adding the parameters:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;--numParallelCollections 1&lt;/li&gt;
	&lt;li&gt;--forceTableScan&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;However, mongodump produced so much load, that we couldn&apos;t even connect to the primary. It failed with:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; PRO [10:59][root@mongo-india01-02.db00.pro06:/home/enrico.rabeneck]# &#10008; mongo&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;MongoDB shell version v4.0.13&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;connecting to: mongodb://127.0.0.1:27017/?gssapiServiceName=mongodb&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2020-08-02T10:59:37.278+0200 E QUERY    [js] Error: couldn&apos;t connect to server 127.0.0.1:27017, connection attempt failed: SocketException: Error connecting to 127.0.0.1:27017 :: caused by :: Connection timed out :&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;connect@src/mongo/shell/mongo.js:344:17&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;@(connect):2:6&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;exception: connect failed&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;As soon as we were able to connect to the Primary, we stepped it down:&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;india01:PRIMARY&amp;gt; rs.printSlaveReplicationInfo()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;source: mongo-india01-01.db00.pro05.eu.idealo.com:27017&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	syncedTo: Sun Aug 02 2020 11:06:48 GMT+0200 (CEST)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	1 secs (0 hrs) behind the primary &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;source: mongo-india01-03.db00.pro07.eu.idealo.com:27017&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	syncedTo: Sun Aug 02 2020 11:06:49 GMT+0200 (CEST)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	0 secs (0 hrs) behind the primary &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;india01:PRIMARY&amp;gt; rs.stepDown()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;&lt;br/&gt;
Database clients could then recover but mongodump was interupted and terminated, so we don&apos;t have a complete backup of this day.&lt;/p&gt;

&lt;p&gt;I&apos;ve uploaded again log and diagnostic files for your investigation. &lt;br/&gt;
Due to these performance issues, we are unable to backup this replica set.&lt;br/&gt;
Any help is greatly appreciated.&lt;/p&gt;
</comment>
                            <comment id="3316822" author="kay.agahd@idealo.de" created="Sat, 1 Aug 2020 20:08:49 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=dmitry.agranat&quot; class=&quot;user-hover&quot; rel=&quot;dmitry.agranat&quot;&gt;dmitry.agranat&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;the same server had again severe problems while mongodump was running. This time, mongod did not crash but it was so slow that it has been declared DEAD and stepped down to Secondary. This time, there were &lt;b&gt;no&lt;/b&gt; mongodump related error messages in the syslog like the last time:&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Jul 22 18:56:19 mongo-india01-03 kernel: [23647675.428924] INFO: task mongodump:274200 blocked for more than 120 seconds.&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Jul 22 18:56:19 mongo-india01-03 kernel: [23647675.428945]       Not tainted 4.9.0-11-amd64 #1 Debian 4.9.189-3&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Jul 22 18:56:19 mongo-india01-03 kernel: [23647675.428959] &quot;echo 0 &amp;gt; /proc/sys/kernel/hung_task_timeout_secs&quot; disables this message.&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Jul 22 18:56:19 mongo-india01-03 kernel: [23647675.428976] mongodump       D    0 274200 273730 0x00000080&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;&lt;br/&gt;
However, this time we have more verbose mongod logs than the last time but all I see is that many queries were very slow when mongodump started (2020-08-01 10:05:01 Germany/Berlin). Did mongodump block them all? However, other mongodb replSet&apos;s having the same hardware and configuration don&apos;t have this problem when mongodump is running. mongodump was connected to mongo-india01-03.db00.pro07. &lt;/p&gt;

&lt;p&gt;I&apos;ve uploaded all mongod log and diagnostic files of this server to the upload location associated to this jira ticket. I&apos;ll upload the same for the other 2 mongodb servers belonging to the same replSet.&lt;/p&gt;

&lt;p&gt;If you prefer that I open a new ticket to follow up this issue, just tell me please.&lt;/p&gt;

&lt;p&gt;Thanks for your investigation!&lt;/p&gt;
</comment>
                            <comment id="3308718" author="dmitry.agranat" created="Tue, 28 Jul 2020 04:52:59 +0000"  >&lt;p&gt;Sounds good &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=kay.agahd%40idealo.de&quot; class=&quot;user-hover&quot; rel=&quot;kay.agahd@idealo.de&quot;&gt;kay.agahd@idealo.de&lt;/a&gt;, I will go ahead and close this case as &quot;Works as expected&quot; as we did not find any MongoDB&apos;s fault + your mention of faulty infrastructure.&lt;/p&gt;

&lt;p&gt;If this happens again, apart from the usual data, please also grab messages, syslog, dmesg logs.&lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
Dima&lt;/p&gt;</comment>
                            <comment id="3307699" author="kay.agahd@idealo.de" created="Mon, 27 Jul 2020 16:35:26 +0000"  >&lt;p&gt;Hi Dima,&lt;/p&gt;

&lt;p&gt;thanks for your suggestions (all known already though). &lt;br/&gt;
So I guess your investigation ends here due to the low log level. I understand this, it&apos;s not a problem. &lt;br/&gt;
I think the reason of the crash was our NFS mounted backup server, which did not respond during more than 120 seconds. However, this should not crash mongodb nor the server itself. Let&apos;s assume this was an exception which shouldn&apos;t have happened.&lt;br/&gt;
If it should happen again, we&apos;ll have more verbose log to let you investigate the case.&lt;/p&gt;</comment>
                            <comment id="3306923" author="dmitry.agranat" created="Mon, 27 Jul 2020 10:51:43 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=kay.agahd%40idealo.de&quot; class=&quot;user-hover&quot; rel=&quot;kay.agahd@idealo.de&quot;&gt;kay.agahd@idealo.de&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;For the sake of this investigation, let&apos;s start by not suppressing MongoDB logs (by removing the quiet:true option), we might need to increase log level during this investigation based on the data we&apos;ll see in unsuppressed logs. For the fresh data collected, please mention timestamps of &lt;tt&gt;mongodump&lt;/tt&gt; start.&lt;/p&gt;

&lt;p&gt;As for a best backup approach for your cluster/workload, I invite you to ask our community by posting on the &lt;a href=&quot;http://community.mongodb.com&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;MongoDB Community Forums&lt;/a&gt; or on &lt;a href=&quot;https://stackoverflow.com/questions/tagged/mongodb&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;Stack Overflow with the &lt;tt&gt;mongodb&lt;/tt&gt; tag&lt;/a&gt;. Just to mention a few points about &lt;tt&gt;mongodump&lt;/tt&gt; which you might want to consider:&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;&lt;tt&gt;mongodump&lt;/tt&gt; will be efficient for backing up small data sets and is not ideal for capturing backups of larger systems (wherein data and indexes do not fit entirely into memory).&lt;/li&gt;
	&lt;li&gt;&lt;tt&gt;mongodump&lt;/tt&gt; and &lt;tt&gt;mongorestore&lt;/tt&gt; operate by interacting with a running &lt;tt&gt;mongod&lt;/tt&gt; instance, hence, they can impact the performance of your running database.&lt;/li&gt;
	&lt;li&gt;&lt;tt&gt;mongodump&lt;/tt&gt; only captures the documents in the database in its backup data and does not include index data. &lt;tt&gt;mongorestore&lt;/tt&gt; requires that &lt;tt&gt;mongod&lt;/tt&gt; then rebuild the indexes after restoring data.&lt;/li&gt;
	&lt;li&gt;If there is any activity going on the database between when the &lt;tt&gt;mongodump&lt;/tt&gt; and when the &lt;tt&gt;mongorestore&lt;/tt&gt; is performed, the new updates to the &lt;tt&gt;mongod&lt;/tt&gt; will need to be accounted for on the new setup.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Thanks,&lt;br/&gt;
Dima&lt;/p&gt;</comment>
                            <comment id="3306613" author="kay.agahd@idealo.de" created="Sun, 26 Jul 2020 22:33:13 +0000"  >&lt;p&gt;Hi Dima,&lt;/p&gt;

&lt;p&gt;our network maximum throughput is&#160;100-Gbit/s. In praxis, our backup server writes more than 600 MiB/sec to disk.&lt;/p&gt;

&lt;p&gt;Thanks for the pointing out the quiet:true option. I think this has been set because the logs grew too fast, especially due to connection attempts. We know that mongodb allows to set different log levels for different components but if we decrease some of them, it might be that exactly these will be required for bug analysis. Can you suggest a well balanced settings for logging?&lt;/p&gt;

&lt;p&gt;Concerning mogodump, what else do you suggest to do daily backup? Do you suggest only paid (cloud) services such as MongoDB Atlas, Cloud Manager or Ops Manager? If it limits to these, mongodb develops in the wrong direction as Oracle did at the time. Time to look for other alternatives then.&lt;/p&gt;</comment>
                            <comment id="3306364" author="dmitry.agranat" created="Sun, 26 Jul 2020 10:05:21 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=kay.agahd%40idealo.de&quot; class=&quot;user-hover&quot; rel=&quot;kay.agahd@idealo.de&quot;&gt;kay.agahd@idealo.de&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;I had a look at the uploaded data. It looks like you have experienced some sort of a network issue during this event, for example:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Error sending response to client: SocketException: Broken pipe. Ending connection from&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Could you clarify what is your network maximum throughput?&lt;/p&gt;

&lt;p&gt;However, since you use &lt;tt&gt;quiet:true&lt;/tt&gt; option, it is not really possible to track down this issue. As per our &lt;a href=&quot;https://docs.mongodb.com/manual/reference/configuration-options/index.html#systemLog.quiet&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;documentation&lt;/a&gt;:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;systemLog.quiet is not recommended for production systems as it may make tracking problems during particular connections much more difficult.&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;In addition, could you also elaborate what is the reason you are running &lt;tt&gt;mongodump&lt;/tt&gt; as your backup method? By design, &lt;tt&gt;mongodump&lt;/tt&gt; will push the working set out of memory, the same working set you will need to read back right away. &lt;/p&gt;</comment>
                            <comment id="3298174" author="kay.agahd@idealo.de" created="Thu, 23 Jul 2020 08:53:51 +0000"  >&lt;p&gt;Hi Dima,&lt;/p&gt;

&lt;p&gt;I uploaded mongod log and diagnostic files of all 3 servers to the upload loacation.&lt;br/&gt;
The start of mongodump was 18:45, timezone Berlin/Germany. I think that is the first log entry related to mongodump:&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2020-07-22T18:45:02.401+0200 I COMMAND  [conn57730] command ipc-catalog.product command: getMore { getMore: 139689230911, collection: &quot;product&quot;, $db: &quot;ipc-catalog&quot; } originatingCommand: { find: &quot;product&quot;, hint: { _id: 1 }, skip: 0, $readPreference: { mode: &quot;secondaryPreferred&quot; }, $db: &quot;ipc-catalog&quot; } planSummary: IXSCAN { _id: 1 } cursorid:139689230911 keysExamined:8946 docsExamined:8946 numYields:69 nreturned:8945 reslen:16776504 locks:{ Global: { acquireCount: { r: 70 } }, Database: { acquireCount: { r: 70 } }, Collection: { acquireCount: { r: 70 } } } storage:{ data: { bytesRead: 107712716, timeReadingMicros: 202868 } } protocol:op_query 254ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;&lt;br/&gt;
The hostname of the member where mongodump was running is mongo-india01-03. It was primary at the time.&lt;br/&gt;
After the crash, mongo-india01-02 took over and is still primary.&lt;br/&gt;
Thanks for your investigation.&lt;/p&gt;</comment>
                            <comment id="3298160" author="dmitry.agranat" created="Thu, 23 Jul 2020 08:17:06 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=kay.agahd%40idealo.de&quot; class=&quot;user-hover&quot; rel=&quot;kay.agahd@idealo.de&quot;&gt;kay.agahd@idealo.de&lt;/a&gt;,&lt;/p&gt;

&lt;p&gt;I&apos;ve created a secure &lt;a href=&quot;https://10gen-httpsupload.s3.amazonaws.com/upload_forms/e57f2f42-c830-494f-bc43-9367e7ac10f7.html&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;upload portal&lt;/a&gt; for you.&lt;/p&gt;

&lt;p&gt;I have one question at this point, on which member mongodump was running (plus mention the start time of mongodump + timezone).&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Dima&lt;/p&gt;
</comment>
                            <comment id="3297715" author="kay.agahd@idealo.de" created="Wed, 22 Jul 2020 21:42:50 +0000"  >&lt;p&gt;It could be related to our backup server which is a NFS mounted device. I found this:&lt;/p&gt;
&lt;blockquote&gt;&lt;p&gt;NFS seems to be a common culprit, probably because it&apos;s good at filling the writeback cache, something which implies blocking while writeback happens - which is likely to block various things related to the same filesystem.&lt;/p&gt;&lt;/blockquote&gt;
&lt;p&gt;Source: &lt;a href=&quot;https://helpful.knobs-dials.com/index.php/INFO:_task_blocked_for_more_than_120_seconds&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://helpful.knobs-dials.com/index.php/INFO:_task_blocked_for_more_than_120_seconds&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;However, this should not crash the server, shouldn&apos;t it?&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>12.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Thu, 23 Jul 2020 08:17:06 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        3 years, 27 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            3 years, 27 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>dmitry.agranat@mongodb.com</customfieldvalue>
            <customfieldvalue>kay.agahd@idealo.de</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hxw6wn:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hxj1br:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[dmitry.agranat@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hxvt5z:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>