<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:47:03 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-37795] WiredTiger again uses way more memory than instructed by cache limit in 3.6</title>
                <link>https://jira.mongodb.org/browse/SERVER-37795</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;As I said in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17424?focusedCommentId=2044120&amp;amp;page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-2044120&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;https://jira.mongodb.org/browse/SERVER-17424?focusedCommentId=2044120&amp;amp;page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-2044120&lt;/a&gt; I noticed yet another regression in 3.6 about memory usage this time (after the secondary CPU &#215; 4 increase and the IO usage &#215; 5 increase), I now have almost twice the configured cacheSize used by mongo, and it increased a lot since 3.4.15:&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/199759/199759_image-2018-10-29-09-22-50-604.png&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;I first upgraded to 3.6.7 and then 3.6.8 after &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-35958?focusedCommentId=2007204&amp;amp;page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-2007204&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;https://jira.mongodb.org/browse/SERVER-35958?focusedCommentId=2007204&amp;amp;page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-2007204&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;My configured max cache is 10G (server has 32G RAM):&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;rs0:PRIMARY&amp;gt; db.serverStatus().version&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;3.6.8&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;rs0:PRIMARY&amp;gt; db.serverStatus().wiredTiger.cache[&quot;maximum bytes configured&quot;]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;10737418240&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;And my actual usage is 19G RES (21G VIRT):&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;rs0:PRIMARY&amp;gt; db.serverStatus().tcmalloc.tcmalloc.formattedString&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;------------------------------------------------&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;MALLOC:    16556355400 (15789.4 MiB) Bytes in use by application&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;MALLOC: +   3597598720 ( 3430.9 MiB) Bytes in page heap freelist&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;MALLOC: +    227350168 (  216.8 MiB) Bytes in central cache freelist&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;MALLOC: +       884736 (    0.8 MiB) Bytes in transfer cache freelist&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;MALLOC: +    290449952 (  277.0 MiB) Bytes in thread cache freelists&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;MALLOC: +     67035392 (   63.9 MiB) Bytes in malloc metadata&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;MALLOC:   ------------&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;MALLOC: =  20739674368 (19778.9 MiB) Actual memory used (physical + swap)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;MALLOC: +   1484984320 ( 1416.2 MiB) Bytes released to OS (aka unmapped)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;MALLOC:   ------------&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;MALLOC: =  22224658688 (21195.1 MiB) Virtual address space used&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;MALLOC:&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;MALLOC:         286938              Spans in use&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;MALLOC:            886              Thread heaps in use&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;MALLOC:           4096              Tcmalloc page size&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;------------------------------------------------&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Here is the full serverStatus:  &lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/attachment/199758/199758_server_status.json&quot; title=&quot;server_status.json attached to SERVER-37795&quot;&gt;server_status.json&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.mongodb.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt; &lt;/p&gt;</description>
                <environment></environment>
        <key id="626084">SERVER-37795</key>
            <summary>WiredTiger again uses way more memory than instructed by cache limit in 3.6</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="13202">Works as Designed</resolution>
                                        <assignee username="alexander.gorrod@mongodb.com">Alexander Gorrod</assignee>
                                    <reporter username="bigbourin@gmail.com">Adrien Jarthon</reporter>
                        <labels>
                            <label>dmd-perf</label>
                            <label>storage-engines</label>
                    </labels>
                <created>Mon, 29 Oct 2018 08:29:18 +0000</created>
                <updated>Fri, 27 Oct 2023 13:53:26 +0000</updated>
                            <resolved>Thu, 22 Nov 2018 02:45:30 +0000</resolved>
                                    <version>3.6.7</version>
                    <version>3.6.8</version>
                                                    <component>WiredTiger</component>
                                        <votes>0</votes>
                                    <watches>18</watches>
                                                                                                                <comments>
                            <comment id="2069689" author="alexander.gorrod" created="Thu, 22 Nov 2018 02:45:11 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bigbourin%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;bigbourin@gmail.com&quot;&gt;bigbourin@gmail.com&lt;/a&gt; sorry the 3.6 release of MongoDB had worse performance characteristics for your application. Thanks very much for your help in understanding and addressing those issues in more recent versions of MongoDB. I&apos;m going to close this ticket since I don&apos;t believe we can do any further work to change the behavior you report. If either the workaround or upgrading don&apos;t help please do let us know and we will investigate further.&lt;/p&gt;</comment>
                            <comment id="2065969" author="bigbourin@gmail.com" created="Mon, 19 Nov 2018 09:25:04 +0000"  >&lt;p&gt;Haha ok, well considering the amount of regressions there was in 3.6 I&apos;m probably not gonna install 4.0 anytime soon. I just lost my PSA cluster thanks to &lt;a href=&quot;https://docs.mongodb.com/manual/reference/read-concern-majority/#disable-read-concern-majority&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://docs.mongodb.com/manual/reference/read-concern-majority/#disable-read-concern-majority&lt;/a&gt;&#160;(as many other people will, good job about that) so I&apos;ll try to give my customers a bit a stability for now.&lt;/p&gt;</comment>
                            <comment id="2061112" author="donald.anderson" created="Wed, 14 Nov 2018 03:11:36 +0000"  >&lt;p&gt;Hello &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bigbourin%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;bigbourin@gmail.com&quot;&gt;bigbourin@gmail.com&lt;/a&gt;, I have worked on some of the cursor issues at a couple layers in the MongoDB stack. &#160;It looks like your path to upgrade should be to first use 3.6.8 with &lt;tt&gt;mongod --setParameter wiredTigerCursorCacheSize=0&lt;/tt&gt; as &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=michael.cahill&quot; class=&quot;user-hover&quot; rel=&quot;michael.cahill&quot;&gt;michael.cahill&lt;/a&gt; suggested. &#160;As for getting to 4.0, we have identified a different issue in 4.0.3 that has a different memory growth problem. That problem is in the WiredTiger cursor layer and is exacerbated for use cases with many updates of large records. A coding fix has been made (&lt;a href=&quot;https://jira.mongodb.org/browse/WT-4418&quot; title=&quot;Don&amp;#39;t keep key/value memory buffers allocated for cached cursors&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-4418&quot;&gt;&lt;del&gt;WT-4418&lt;/del&gt;&lt;/a&gt;) and should be in an upcoming release. Until you get a release that includes that fix, you should continue to run with &lt;tt&gt;mongod --setParameter wiredTigerCursorCacheSize=0&lt;/tt&gt;, as that works around the 4.0 problem as well. I&apos;ve verified that memory is pretty stable with 4.0.3 with this parameter - you should as well! Finally, after you have a 4.0.x release with the fix, you can drop the extra parameter. Running with the parameter on makes cursor operations a bit slower, and it&apos;s possible it may not be so noticeable for your use case.&lt;/p&gt;</comment>
                            <comment id="2055407" author="bigbourin@gmail.com" created="Thu, 8 Nov 2018 08:29:37 +0000"  >&lt;p&gt;I confirm that with the &lt;tt&gt;wiredTigerCursorCacheSize&lt;/tt&gt; change, the memory is back to stable levels (11G).&lt;br/&gt;
Shall I assume that no improvement will be backported to 3.6? am I supposed to be able to remove this parameter when upgrading to 4.0?&lt;/p&gt;</comment>
                            <comment id="2051398" author="bigbourin@gmail.com" created="Mon, 5 Nov 2018 10:08:37 +0000"  >&lt;p&gt;Hi Michael, thanks for the details, I&apos;ve just restarted with the option to see how far it goes now.&lt;/p&gt;

&lt;p&gt;About the improvement in 4.0, this is good but with &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-35958&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;so many&lt;/a&gt; &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-37233&quot; class=&quot;external-link&quot; rel=&quot;nofollow&quot;&gt;regressions&lt;/a&gt; in 3.6 it&apos;s not easy to upgrade or even test 4.0&lt;br/&gt;
The one I explained in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-35958&quot; title=&quot;Big CPU load increase (&#215;4) on secondary by upgrading 3.4.15 &#8594; 3.6.5&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-35958&quot;&gt;&lt;del&gt;SERVER-35958&lt;/del&gt;&lt;/a&gt; (CPU on secondary) is said to be improved in 4.0 too but if I upgrade my secondary to 3.6 it dies in 24h.&lt;br/&gt;
Am I supposed to buy a bigger server just to be able to upgrade through the regressions?&lt;br/&gt;
And if try to upgrade quickly in hope it&apos;ll work better, I&apos;ll be stuck there (between 3.6 and 4.0) without being able to downgrade.&lt;br/&gt;
I can understand it&apos;s cumbersome to backport this fixes but I&apos;m not the only one who will face trouble trying to upgrade through 3.6 if none of them are. If you have no better suggestion I&apos;ll buy a bigger machine for this upgrade but in this case you should at least warn people that updating to 3.6 is dangerous.&lt;/p&gt;</comment>
                            <comment id="2051325" author="michael.cahill" created="Mon, 5 Nov 2018 05:04:51 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bigbourin%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;bigbourin@gmail.com&quot;&gt;bigbourin@gmail.com&lt;/a&gt;, as &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt; explained, the memory MongoDB is using in your case is associated with cursors.  The difference you are seeing in 3.6 is that MongoDB is representing small changes to documents as &quot;modify&quot; operations in WiredTiger, which causes a buffer to be allocated in cursors that read those documents.&lt;/p&gt;

&lt;p&gt;The caching behavior of cursors has been significantly changed in MongoDB 4.0 and above and you should see far fewer active cursors.&lt;/p&gt;

&lt;p&gt;In 3.6, please test with:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongod --setParameter wiredTigerCursorCacheSize=0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;This will disable one layer of cursor caching inside MongoDB.  There is potential for some performance impact, but in our testing that impact has been small.  What you should see is far fewer cursors open, and thus less memory allocated to buffers that hold documents.&lt;/p&gt;</comment>
                            <comment id="2051118" author="bigbourin@gmail.com" created="Sun, 4 Nov 2018 09:50:01 +0000"  >&lt;p&gt;I&apos;m back at 19G now, I&apos;ve just uploaded the following 4 days of diagnostic data (10/31 to 11/04) and the updated log file from 10/29 to 11/04 (now) in case it helps.&lt;/p&gt;</comment>
                            <comment id="2047387" author="bigbourin@gmail.com" created="Wed, 31 Oct 2018 14:03:55 +0000"  >&lt;p&gt;Ok thx &lt;img class=&quot;emoticon&quot; src=&quot;https://jira.mongodb.org/images/icons/emoticons/thumbs_up.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/p&gt;</comment>
                            <comment id="2047385" author="bruce.lucas@10gen.com" created="Wed, 31 Oct 2018 14:01:47 +0000"  >&lt;p&gt;Thanks for the data Adrien. You are correct, the excess allocated memory at this point is sufficient to get a good signal:&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/199990/199990_excess.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;The excess is accounted for by the following allocation sites, all in __wt_modify_apply or __wt_modify_apply_api. I imagine this may be either scratch memory that is kept around too long, or memory in cache that is not properly accounted for. I&apos;ll pass this off to a storage expert to take a look.&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2018-10-29T22:34:35.038+0100 I -        [ftdc] heapProfile stack1811: { 0: &quot;tc_realloc&quot;, 1: &quot;0x5621b07d9e17&quot;, 2: &quot;__wt_buf_grow_worker&quot;, 3: &quot;0x5621b0813d09&quot;, 4: &quot;__wt_modify_apply&quot;, 5: &quot;__wt_value_return_upd&quot;, 6: &quot;__wt_value_return&quot;, 7: &quot;__wt_btcur_next&quot;, 8: &quot;0x5621b0879c63&quot;, 9: &quot;mongo::WiredTigerRecordStoreCursorBase::next&quot;, 10: &quot;mongo::CollectionScan::doWork&quot;, 11: &quot;mongo::PlanStage::work&quot;, 12: &quot;mongo::PlanExecutor::getNextImpl&quot;, 13: &quot;mongo::PlanExecutor::getNext&quot;, 14: &quot;0x5621b09b8c8a&quot;, 15: &quot;mongo::BasicCommand::enhancedRun&quot;, 16: &quot;mongo::Command::publicRun&quot;, 17: &quot;0x5621b096676a&quot;, 18: &quot;0x5621b0967afb&quot;, 19: &quot;mongo::ServiceEntryPointMongod::handleRequest&quot;, 20: &quot;mongo::ServiceStateMachine::_processMessage&quot;, 21: &quot;mongo::ServiceStateMachine::_runNextInGuard&quot;, 22: &quot;0x5621b0973c41&quot;, 23: &quot;mongo::transport::ServiceExecutorSynchronous::schedule&quot;, 24: &quot;mongo::ServiceStateMachine::_scheduleNextWithGuard&quot;, 25: &quot;mongo::ServiceStateMachine::_sourceCallback&quot;, 26: &quot;mongo::ServiceStateMachine::_sourceMessage&quot;, 27: &quot;mongo::ServiceStateMachine::_runNextInGuard&quot;, 28: &quot;0x5621b0973c41&quot;, 29: &quot;0x5621b1876545&quot;, 30: &quot;0x5621b1e352c4&quot;, 31: &quot;0x7f55dfee76ba&quot;, 32: &quot;clone&quot; }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2018-10-29T22:33:49.075+0100 I -        [ftdc] heapProfile stack1470: { 0: &quot;tc_realloc&quot;, 1: &quot;0x5621b07d9e17&quot;, 2: &quot;__wt_buf_grow_worker&quot;, 3: &quot;0x5621b0813d09&quot;, 4: &quot;__wt_modify_apply_api&quot;, 5: &quot;__wt_btcur_modify&quot;, 6: &quot;0x5621b08789ee&quot;, 7: &quot;mongo::WiredTigerRecordStore::updateWithDamages&quot;, 8: &quot;mongo::CollectionImpl::updateDocumentWithDamages&quot;, 9: &quot;mongo::UpdateStage::transformAndUpdate&quot;, 10: &quot;mongo::UpdateStage::doWork&quot;, 11: &quot;mongo::PlanStage::work&quot;, 12: &quot;mongo::PlanExecutor::getNextImpl&quot;, 13: &quot;mongo::PlanExecutor::getNext&quot;, 14: &quot;mongo::PlanExecutor::executePlan&quot;, 15: &quot;mongo::performUpdates&quot;, 16: &quot;0x5621b0a23330&quot;, 17: &quot;0x5621b0a1cdd8&quot;, 18: &quot;mongo::Command::publicRun&quot;, 19: &quot;0x5621b0966e53&quot;, 20: &quot;0x5621b0967afb&quot;, 21: &quot;mongo::ServiceEntryPointMongod::handleRequest&quot;, 22: &quot;mongo::ServiceStateMachine::_processMessage&quot;, 23: &quot;mongo::ServiceStateMachine::_runNextInGuard&quot;, 24: &quot;0x5621b0973c41&quot;, 25: &quot;mongo::transport::ServiceExecutorSynchronous::schedule&quot;, 26: &quot;mongo::ServiceStateMachine::_scheduleNextWithGuard&quot;, 27: &quot;mongo::ServiceStateMachine::_sourceCallback&quot;, 28: &quot;mongo::ServiceStateMachine::_sourceMessage&quot;, 29: &quot;mongo::ServiceStateMachine::_runNextInGuard&quot;, 30: &quot;0x5621b0973c41&quot;, 31: &quot;0x5621b1876545&quot;, 32: &quot;0x5621b1e352c4&quot;, 33: &quot;0x7f55dfee76ba&quot;, 34: &quot;clone&quot; }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2018-10-29T22:35:02.019+0100 I -        [ftdc] heapProfile stack1908: { 0: &quot;tc_realloc&quot;, 1: &quot;0x5621b07d9e17&quot;, 2: &quot;__wt_buf_grow_worker&quot;, 3: &quot;0x5621b0813d09&quot;, 4: &quot;__wt_modify_apply&quot;, 5: &quot;__wt_value_return_upd&quot;, 6: &quot;__wt_value_return&quot;, 7: &quot;__wt_btcur_next&quot;, 8: &quot;0x5621b0879c63&quot;, 9: &quot;mongo::WiredTigerRecordStoreCursorBase::next&quot;, 10: &quot;mongo::CollectionScan::doWork&quot;, 11: &quot;mongo::PlanStage::work&quot;, 12: &quot;mongo::PlanExecutor::getNextImpl&quot;, 13: &quot;mongo::PlanExecutor::getNext&quot;, 14: &quot;0x5621b09b8c8a&quot;, 15: &quot;mongo::BasicCommand::enhancedRun&quot;, 16: &quot;mongo::Command::publicRun&quot;, 17: &quot;0x5621b096676a&quot;, 18: &quot;0x5621b0967afb&quot;, 19: &quot;mongo::ServiceEntryPointMongod::handleRequest&quot;, 20: &quot;mongo::ServiceStateMachine::_processMessage&quot;, 21: &quot;mongo::ServiceStateMachine::_runNextInGuard&quot;, 22: &quot;0x5621b0973c41&quot;, 23: &quot;mongo::transport::ServiceExecutorSynchronous::schedule&quot;, 24: &quot;mongo::ServiceStateMachine::_scheduleNextWithGuard&quot;, 25: &quot;mongo::ServiceStateMachine::_sourceCallback&quot;, 26: &quot;mongo::ServiceStateMachine::_sourceMessage&quot;, 27: &quot;mongo::ServiceStateMachine::_runNextInGuard&quot;, 28: &quot;0x5621b0973c41&quot;, 29: &quot;0x5621b1876545&quot;, 30: &quot;0x5621b1e352c4&quot;, 31: &quot;0x7f55dfee76ba&quot;, 32: &quot;clone&quot; }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2018-10-29T22:33:43.038+0100 I -        [ftdc] heapProfile stack1327: { 0: &quot;tc_realloc&quot;, 1: &quot;0x5621b07d9e17&quot;, 2: &quot;__wt_buf_grow_worker&quot;, 3: &quot;0x5621b0813d09&quot;, 4: &quot;__wt_modify_apply&quot;, 5: &quot;__wt_value_return_upd&quot;, 6: &quot;__wt_value_return&quot;, 7: &quot;__wt_btcur_search&quot;, 8: &quot;0x5621b087d36d&quot;, 9: &quot;mongo::WiredTigerRecordStore::updateRecord&quot;, 10: &quot;mongo::CollectionImpl::updateDocument&quot;, 11: &quot;mongo::UpdateStage::transformAndUpdate&quot;, 12: &quot;mongo::UpdateStage::doWork&quot;, 13: &quot;mongo::PlanStage::work&quot;, 14: &quot;mongo::PlanExecutor::getNextImpl&quot;, 15: &quot;mongo::PlanExecutor::getNext&quot;, 16: &quot;mongo::PlanExecutor::executePlan&quot;, 17: &quot;mongo::performUpdates&quot;, 18: &quot;0x5621b0a23330&quot;, 19: &quot;0x5621b0a1cdd8&quot;, 20: &quot;mongo::Command::publicRun&quot;, 21: &quot;0x5621b0966e53&quot;, 22: &quot;0x5621b0967afb&quot;, 23: &quot;mongo::ServiceEntryPointMongod::handleRequest&quot;, 24: &quot;mongo::ServiceStateMachine::_processMessage&quot;, 25: &quot;mongo::ServiceStateMachine::_runNextInGuard&quot;, 26: &quot;0x5621b0973c41&quot;, 27: &quot;mongo::transport::ServiceExecutorSynchronous::schedule&quot;, 28: &quot;mongo::ServiceStateMachine::_scheduleNextWithGuard&quot;, 29: &quot;mongo::ServiceStateMachine::_sourceCallback&quot;, 30: &quot;mongo::ServiceStateMachine::_sourceMessage&quot;, 31: &quot;mongo::ServiceStateMachine::_runNextInGuard&quot;, 32: &quot;0x5621b0973c41&quot;, 33: &quot;0x5621b1876545&quot;, 34: &quot;0x5621b1e352c4&quot;, 35: &quot;0x7f55dfee76ba&quot;, 36: &quot;clone&quot; }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2018-10-29T22:33:46.022+0100 I -        [ftdc] heapProfile stack1388: { 0: &quot;tc_realloc&quot;, 1: &quot;0x5621b07d9e17&quot;, 2: &quot;__wt_buf_grow_worker&quot;, 3: &quot;0x5621b0813d09&quot;, 4: &quot;__wt_modify_apply&quot;, 5: &quot;__wt_value_return_upd&quot;, 6: &quot;__wt_value_return&quot;, 7: &quot;__wt_btcur_search&quot;, 8: &quot;0x5621b087d36d&quot;, 9: &quot;mongo::WiredTigerRecordStoreCursorBase::seekExact&quot;, 10: &quot;mongo::WorkingSetCommon::fetch&quot;, 11: &quot;mongo::IDHackStage::doWork&quot;, 12: &quot;mongo::PlanStage::work&quot;, 13: &quot;mongo::UpdateStage::doWork&quot;, 14: &quot;mongo::PlanStage::work&quot;, 15: &quot;mongo::PlanExecutor::getNextImpl&quot;, 16: &quot;mongo::PlanExecutor::getNext&quot;, 17: &quot;mongo::PlanExecutor::executePlan&quot;, 18: &quot;mongo::performUpdates&quot;, 19: &quot;0x5621b0a23330&quot;, 20: &quot;0x5621b0a1cdd8&quot;, 21: &quot;mongo::Command::publicRun&quot;, 22: &quot;0x5621b0966e53&quot;, 23: &quot;0x5621b0967afb&quot;, 24: &quot;mongo::ServiceEntryPointMongod::handleRequest&quot;, 25: &quot;mongo::ServiceStateMachine::_processMessage&quot;, 26: &quot;mongo::ServiceStateMachine::_runNextInGuard&quot;, 27: &quot;0x5621b0973c41&quot;, 28: &quot;mongo::transport::ServiceExecutorSynchronous::schedule&quot;, 29: &quot;mongo::ServiceStateMachine::_scheduleNextWithGuard&quot;, 30: &quot;mongo::ServiceStateMachine::_sourceCallback&quot;, 31: &quot;mongo::ServiceStateMachine::_sourceMessage&quot;, 32: &quot;mongo::ServiceStateMachine::_runNextInGuard&quot;, 33: &quot;0x5621b0973c41&quot;, 34: &quot;0x5621b1876545&quot;, 35: &quot;0x5621b1e352c4&quot;, 36: &quot;0x7f55dfee76ba&quot;, 37: &quot;clone&quot; }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2018-10-29T22:34:04.044+0100 I -        [ftdc] heapProfile stack1648: { 0: &quot;tc_realloc&quot;, 1: &quot;0x5621b07d9e17&quot;, 2: &quot;__wt_buf_grow_worker&quot;, 3: &quot;0x5621b0813d09&quot;, 4: &quot;__wt_modify_apply_api&quot;, 5: &quot;__wt_btcur_modify&quot;, 6: &quot;0x5621b08789ee&quot;, 7: &quot;mongo::WiredTigerRecordStore::updateWithDamages&quot;, 8: &quot;mongo::CollectionImpl::updateDocumentWithDamages&quot;, 9: &quot;mongo::UpdateStage::transformAndUpdate&quot;, 10: &quot;mongo::UpdateStage::doWork&quot;, 11: &quot;mongo::PlanStage::work&quot;, 12: &quot;mongo::PlanExecutor::getNextImpl&quot;, 13: &quot;mongo::PlanExecutor::getNext&quot;, 14: &quot;mongo::PlanExecutor::executePlan&quot;, 15: &quot;mongo::performUpdates&quot;, 16: &quot;0x5621b0a23330&quot;, 17: &quot;0x5621b0a1cdd8&quot;, 18: &quot;mongo::Command::publicRun&quot;, 19: &quot;0x5621b0966e53&quot;, 20: &quot;0x5621b0967afb&quot;, 21: &quot;mongo::ServiceEntryPointMongod::handleRequest&quot;, 22: &quot;mongo::ServiceStateMachine::_processMessage&quot;, 23: &quot;mongo::ServiceStateMachine::_runNextInGuard&quot;, 24: &quot;0x5621b0973c41&quot;, 25: &quot;mongo::transport::ServiceExecutorSynchronous::schedule&quot;, 26: &quot;mongo::ServiceStateMachine::_scheduleNextWithGuard&quot;, 27: &quot;mongo::ServiceStateMachine::_sourceCallback&quot;, 28: &quot;mongo::ServiceStateMachine::_sourceMessage&quot;, 29: &quot;mongo::ServiceStateMachine::_runNextInGuard&quot;, 30: &quot;0x5621b0973c41&quot;, 31: &quot;0x5621b1876545&quot;, 32: &quot;0x5621b1e352c4&quot;, 33: &quot;0x7f55dfee76ba&quot;, 34: &quot;clone&quot; }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="2047154" author="bigbourin@gmail.com" created="Wed, 31 Oct 2018 08:27:51 +0000"  >&lt;p&gt;Ok I&apos;ve restarted the primary on the 29th at 9pm UTC, it&apos;s not back to 19G yet but still 15G which is already much more than 3.4&lt;br/&gt;
I&apos;ve attached the diagnostic data and log file (which contains the heapProfile lines).&lt;/p&gt;</comment>
                            <comment id="2044834" author="bigbourin@gmail.com" created="Mon, 29 Oct 2018 14:40:24 +0000"  >&lt;p&gt;I&apos;ll try to do that&lt;/p&gt;</comment>
                            <comment id="2044660" author="bruce.lucas@10gen.com" created="Mon, 29 Oct 2018 12:53:50 +0000"  >&lt;p&gt;Hi Adrien,&lt;/p&gt;

&lt;p&gt;Thanks for reporting this. The serverStatus output that you uploaded tells us that the memory usage is not due to the WiredTiger cache, but rather is allocated memory outside the cache:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&quot;bytes currently in the cache&quot; :         8593667541,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&quot;current_allocated_bytes&quot; : NumberLong(&quot;16818939408&quot;),&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;The diagnostic data that you uploaded confirms that but in this case it doesn&apos;t tell us much about what is allocating the memory. Over the course of the 4 days covered by the data the amount of excess memory has remained fairly constant, which is also consistent with the graphs that you provided.&lt;/p&gt;

&lt;p&gt;Would you be in a position to enable the built-in heap profiler on an affected node by setting the heapProfilingEnabled parameter to true, e.g. on the command line:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;--setParameter heapProfilingEnabled=true&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;(or equivalent in the config file). Note that this may have some small impact on performance, so please monitor the node(s) where you have enabled this. Please allow maybe a day or two of runtime to get a good signal in the data, then upload both the diagnostic.data and the complete log file(s) since the restart of the node to the portal. (It&apos;s important to have the complete log files because they have the crucial allocating stack information.)&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
 Bruce&lt;/p&gt;</comment>
                            <comment id="2044630" author="bigbourin@gmail.com" created="Mon, 29 Oct 2018 11:56:07 +0000"  >&lt;p&gt;@Eric I&apos;m not saying it&apos;s the WT cache itself but that the totally should not use substantially more than the max WT cache size we can configure.&lt;br/&gt;
About the previous tickets yeah there was &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17424&quot; title=&quot;WiredTiger uses substantially more memory than accounted for by cache&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17424&quot;&gt;&lt;del&gt;SERVER-17424&lt;/del&gt;&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-20306&quot; title=&quot;75% excess memory usage under WiredTiger during stress test&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-20306&quot;&gt;&lt;del&gt;SERVER-20306&lt;/del&gt;&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-22906&quot; title=&quot;MongoD uses excessive memory over and above the WiredTiger cache size&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-22906&quot;&gt;&lt;del&gt;SERVER-22906&lt;/del&gt;&lt;/a&gt;, to name a few.&lt;/p&gt;

&lt;p&gt;So it seems this issue has been present, improved and then regressed.&lt;/p&gt;

&lt;p&gt;@Kelsey I&apos;ve uploaded &lt;tt&gt;web1_diagnostic.data.tgz&lt;/tt&gt; to the portal. BTW the tool you&apos;re using to read these diagnostic data is proprietary I guess? no chance we can use it as end users? It could be pretty useful for monitoring and performance analysis aside from bug reports.&lt;/p&gt;</comment>
                            <comment id="2044615" author="thomas.schubert" created="Mon, 29 Oct 2018 11:07:56 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bigbourin%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;bigbourin@gmail.com&quot;&gt;bigbourin@gmail.com&lt;/a&gt;, would you please upload an archive of the diagnostic.data directory to this &lt;a href=&quot;https://10gen-httpsupload.s3.amazonaws.com/upload_forms/e1aaa660-121b-4f95-9f61-d7c6a30e586a.html&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;upload portal&lt;/a&gt;?&lt;/p&gt;</comment>
                            <comment id="2044578" author="milkie" created="Mon, 29 Oct 2018 09:27:41 +0000"  >&lt;p&gt;I see now &amp;#8211; the ticket &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17424&quot; title=&quot;WiredTiger uses substantially more memory than accounted for by cache&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17424&quot;&gt;&lt;del&gt;SERVER-17424&lt;/del&gt;&lt;/a&gt; had similar symptoms in version 3.0.2, in 2015.  We will investigate this issue.&lt;/p&gt;</comment>
                            <comment id="2044577" author="milkie" created="Mon, 29 Oct 2018 09:23:05 +0000"  >&lt;p&gt;Adrien, how do you know that the physical RAM in use by tcmalloc in the mongod process is all attributable to the WiredTiger page cache?  Also, what do you mean by &quot;again&quot; &amp;#8211; can you link the original ticket, as I&apos;m not familiar with it?&lt;/p&gt;</comment>
                            <comment id="2044564" author="bigbourin@gmail.com" created="Mon, 29 Oct 2018 08:33:14 +0000"  >&lt;p&gt;Sorry it seems my first image upload lacks labels and I can&apos;t edit the description so here is the fixed version:&lt;br/&gt;
&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/199760/199760_mongodb-3.6-memory-regression.png&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;br/&gt;
ps: It&apos;s actually 3.6.7 and then 3.6.8 not 3.6.6.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="629289">SERVER-37926</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="631493">WT-4418</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                                        </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="199990" name="excess.png" size="169859" author="bruce.lucas@mongodb.com" created="Wed, 31 Oct 2018 14:00:56 +0000"/>
                            <attachment id="199759" name="image-2018-10-29-09-22-50-604.png" size="16896" author="bigbourin@gmail.com" created="Mon, 29 Oct 2018 08:23:06 +0000"/>
                            <attachment id="199760" name="mongodb-3.6-memory-regression.png" size="18906" author="bigbourin@gmail.com" created="Mon, 29 Oct 2018 08:32:36 +0000"/>
                            <attachment id="199758" name="server_status.json" size="34324" author="bigbourin@gmail.com" created="Mon, 29 Oct 2018 08:29:07 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>17.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[500A000000cdlsiIAA]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 29 Oct 2018 09:23:05 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        5 years, 12 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            5 years, 12 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>bigbourin@gmail.com</customfieldvalue>
            <customfieldvalue>alexander.gorrod@mongodb.com</customfieldvalue>
            <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>donald.anderson@mongodb.com</customfieldvalue>
            <customfieldvalue>milkie@mongodb.com</customfieldvalue>
            <customfieldvalue>kelsey.schubert@mongodb.com</customfieldvalue>
            <customfieldvalue>michael.cahill@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hubk5r:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hu1tzr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="2623">Storage Engines 2018-11-19</customfieldvalue>
    <customfieldvalue id="2675">Storage Engines 2018-12-03</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[bruce.lucas@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hub6f3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>