<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:14:00 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-27045] Over 40% regression on mongodb using YCSB workload - follow on to server-26674</title>
                <link>https://jira.mongodb.org/browse/SERVER-27045</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;We opened ticket &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-26674&quot; title=&quot;Over 25% regression on mongodb using YCSB workload &quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-26674&quot;&gt;&lt;del&gt;SERVER-26674&lt;/del&gt;&lt;/a&gt; on a performance issue. While 3.4rc3 fixed the issue reported on &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-26674&quot; title=&quot;Over 25% regression on mongodb using YCSB workload &quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-26674&quot;&gt;&lt;del&gt;SERVER-26674&lt;/del&gt;&lt;/a&gt;,  when we tried the same YCSB test with a different set of parameters we are  running into performance problems. &lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;version         Results&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;----------------------------------------&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;3.3.8           302280.29 ops/sec&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;3.4.rc3         130755.69 ops/sec &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;The vmstat with version 3.4rc3 shows high and lows where as with 3.3.8 it is very even.&lt;/p&gt;





</description>
                <environment></environment>
        <key id="332051">SERVER-27045</key>
            <summary>Over 40% regression on mongodb using YCSB workload - follow on to server-26674</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="michael.cahill@mongodb.com">Michael Cahill</assignee>
                                    <reporter username="lilianr@us.ibm.com">Lilian Romero</reporter>
                        <labels>
                    </labels>
                <created>Tue, 15 Nov 2016 18:49:55 +0000</created>
                <updated>Mon, 23 Jan 2017 03:48:48 +0000</updated>
                            <resolved>Mon, 5 Dec 2016 04:28:18 +0000</resolved>
                                    <version>3.4.0-rc3</version>
                                                    <component>Performance</component>
                    <component>WiredTiger</component>
                                        <votes>0</votes>
                                    <watches>15</watches>
                                                                                                                <comments>
                            <comment id="1445151" author="jvf" created="Tue, 29 Nov 2016 20:12:00 +0000"  >&lt;p&gt;Well, 5% is almost within measurement variance.  I ran agan today and got profile, etc data &amp;#8211; I did not see the symptoms from earler (widely varying cpu usage and calls to eviction). Throughput was down a bit because of the data collection, but I think we are ok for now.  One thought, though, is to make the parameters we used to be defaults - perhaps/&lt;/p&gt;

&lt;p&gt;OK to close&lt;/p&gt;</comment>
                            <comment id="1442988" author="alexander.gorrod" created="Mon, 28 Nov 2016 05:03:54 +0000"  >&lt;p&gt;Thanks for the new data &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=jvf&quot; class=&quot;user-hover&quot; rel=&quot;jvf&quot;&gt;jvf&lt;/a&gt;; from the numbers you posted the most recent result is 5% slower than the numbers achieved on the 3.3.8 development release.&lt;/p&gt;

&lt;p&gt;I have analyzed the diagnostic data you uploaded and throughput is very stable on this most recent run. I could not diagnose any further bottle necks related to WiredTiger cache management. There have been a lot of changes that improve correctness and functionality between 3.3.8 and 3.4.0 - as such it would be difficult to isolate the causes of this performance difference. Are you happy for this ticket to be closed now, with the knowledge that MongoDB will continue to improve performance in the future, and that YCSB is one of the benchmarks we use to track that improvement?&lt;/p&gt;</comment>
                            <comment id="1440341" author="jvf" created="Tue, 22 Nov 2016 18:21:59 +0000"  >&lt;p&gt;diagnostic data &lt;/p&gt;</comment>
                            <comment id="1440335" author="jvf" created="Tue, 22 Nov 2016 18:20:15 +0000"  >&lt;p&gt;ran with those changes, improved a little to 286722.60 ops/sec.  Diagnostic data attached.&lt;/p&gt;</comment>
                            <comment id="1439794" author="michael.cahill" created="Tue, 22 Nov 2016 05:41:55 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=jvf&quot; class=&quot;user-hover&quot; rel=&quot;jvf&quot;&gt;jvf&lt;/a&gt; thanks for the additional information.&lt;/p&gt;

&lt;p&gt;A couple of things I noticed about the workload:&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;there appears to be I/O saturation every 30 seconds (on devices &lt;tt&gt;sdl&lt;/tt&gt; through &lt;tt&gt;sdr&lt;/tt&gt;) leading to slow write operations to the filesystem.  These aren&apos;t obviously correlated to operations in MongoDB so are probably caused by Linux flushing dirty pages.  You might want to try setting &lt;tt&gt;vm.dirty_background_ratio = 1&lt;/tt&gt; (or some other small number) to see if that helps smooth out this I/O.&lt;/li&gt;
&lt;/ul&gt;


&lt;ul&gt;
	&lt;li&gt;this YCSB workload is doing random updates with a uniform request distribution.  That means it will attempt to dirty all the data in the collection all the time.  There appears to be about 36GB uncompressed in cache and the default settings try to reduce the amount of dirty data a checkpoint has to process to 5% of the cache size, or 6.4GB in this case.  That scrubbing step is the cause of the remaining time difference compared with 3.3.8.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;You should be able to restore the 3.3.8 behavior with the following settings:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongod --wiredTigerEngineConfigString=&quot;eviction_dirty_trigger=95,eviction_dirty_target=80&quot;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Note that these settings will result in checkpoints taking a long time to complete, leading to additional disk requirements for journal data and slower recovery after an unclean shutdown.  For this workload, they should avoid eviction entirely because the dirty working set will stay in cache.&lt;/p&gt;</comment>
                            <comment id="1439332" author="jvf" created="Mon, 21 Nov 2016 19:08:32 +0000"  >&lt;p&gt;ran today &amp;#8211; same parms, same-ish results 262966.39 ops/sec.  Diagnostic data attached.  &lt;/p&gt;</comment>
                            <comment id="1438521" author="michael.cahill" created="Sun, 20 Nov 2016 22:49:30 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=jvf&quot; class=&quot;user-hover&quot; rel=&quot;jvf&quot;&gt;jvf&lt;/a&gt;, thanks, it&apos;s good to hear that this setting helped.  Can you upload the &lt;tt&gt;diagnostic.data&lt;/tt&gt; from that run so I can see where the remaining 13% is going?&lt;/p&gt;</comment>
                            <comment id="1437925" author="jvf" created="Fri, 18 Nov 2016 19:43:22 +0000"  >&lt;p&gt;I ran with that change and the result improved to 262908.22 ops/sec which is still about 13% down from 3.3.8.  You are certainly  on the right track! &amp;#8211; just a little bit more. Can you suggest  different trigger percentages to experiment with? Other parameters?&lt;/p&gt;</comment>
                            <comment id="1437278" author="michael.cahill" created="Fri, 18 Nov 2016 03:15:08 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=lilianr%40us.ibm.com&quot; class=&quot;user-hover&quot; rel=&quot;lilianr@us.ibm.com&quot;&gt;lilianr@us.ibm.com&lt;/a&gt;, thank you for the data, I have been looking into this today.&lt;/p&gt;

&lt;p&gt;This reduction in throughput is caused by some changes to limit how much of the WiredTiger cache is permitted to be dirty by default.  In the 3.3.8 runs, much of the cache is dirty, leading to variable memory use and checkpoints taking a very long time (over 10 minutes each).  The 3.4.0-rc4 runs here are behaving better in terms of memory use and checkpoints completing, but the stalls are unexpected and unacceptable.&lt;/p&gt;

&lt;p&gt;If you are able to run further tests with 3.4.0 release candidates, can you please try starting MongoDB with the following setting:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongod --wiredTigerEngineConfigString=&quot;eviction_dirty_trigger=95&quot; ...&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;That should largely restore the 3.3.8 behavior: if you can run that test, please let me know how the results look.&lt;/p&gt;

&lt;p&gt;I have reproduced this effect and will work to improve the behavior with the default settings.  I will let you know when I have more information.&lt;/p&gt;</comment>
                            <comment id="1435819" author="lilianr@us.ibm.com" created="Wed, 16 Nov 2016 18:34:23 +0000"  >&lt;p&gt;The config file used is the default.&lt;br/&gt;
Here is portion of the vmstat for 3.4 rc3.  As mentioned previously in version 3.3.8, cpu utilization is very stable where as in 3.4 rc3 there are high and lows.  The attachment&lt;br/&gt;
shows the pstack for the low and the high. Note that in the low case over 90 threads are in wait from __wt_cache_eviction_worker where as in higher utilization there is only 3.&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;106  0      0 152681600 138432 51564928    0    0     8 49440 418499 1019938 51 12 37  0  0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;108  0      0 152542144 138432 51686464    0    0     4 47808 420635 1026560 52 12 36  0  0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;101  0      0 152421120 138432 51808192    0    0     0 54824 420483 1055053 51 12 36  0  0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;112  0      0 152278016 138432 51908544    0    0     0 1281200 414543 1067763 52 12 36  0  0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;77  6      0 152150336 138432 52009920    0    0     0 3136104 432092 1070823 51 13 35  1  0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;100  0      0 152058240 138432 52107008    0    0     0 2584652 293156 510549 21  6 72  0  0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;64  0      0 151957888 138432 52203840    0    0     0 673368 340965 716791 33 11 57  0  0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;40  0      0 151799616 138432 52360256    0    0     0 36628 370159 820120 39 12 49  0  0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa st&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;120  0      0 151639616 138432 52519616    0    0     0 32972 366290 830841 39 11 50  0  0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;103  0      0 151487872 138432 52669248    0    0     0 35640 364529 822919 37 11 52  0  0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;54  0      0 151355328 138432 52802240    0    0     0 35040 362368 813807 37 10 53  0  0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;107  1      0 151219968 138432 52937792    0    0     0 31780 360528 813909 36 11 52  0  0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;12  0      0 151118272 138432 53038784    0    0     0 23144 273532 639754 27  8 66  0  0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; 4  0      0 151095936 138432 53062336    0    0     0  8632 181240 400825 12  4 84  0  0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; 3  0      0 151087680 138432 53070336    0    0     0  8832 174087 334576 11  4 85  0  0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; 4  0      0 151078336 138432 53079424    0    0     0  7836 181645 358191 10  4 86  0  0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; 3  0      0 151069568 138432 53087808    0    0     0  8972 169224 321014 10  4 86  0  0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="1435189" author="lilianr@us.ibm.com" created="Wed, 16 Nov 2016 02:17:47 +0000"  >&lt;p&gt;The attachment contains the diagnostic data for version 3.3.8&lt;/p&gt;</comment>
                            <comment id="1434999" author="lilianr@us.ibm.com" created="Tue, 15 Nov 2016 21:40:58 +0000"  >&lt;p&gt;The attachment contains the diagnostics data for 3.4.cr3&lt;/p&gt;</comment>
                            <comment id="1434915" author="dan@10gen.com" created="Tue, 15 Nov 2016 20:30:30 +0000"  >&lt;p&gt;Hi Lillian, thanks for the report.  A couple questions:&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;What is the configuration you are running your mongodb with (either startup params or the config file used)&lt;/li&gt;
	&lt;li&gt;Can you attach the files from the &lt;tt&gt;diagnostic.data&lt;/tt&gt; directory in the dbpath of a run with 3.3.8 and 3.4.0-rc3?&lt;/li&gt;
&lt;/ul&gt;
</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="334440">WT-3040</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="144541" name="2016-11-21T17-40-57Z.tar.gz" size="1233116" author="jvf" created="Mon, 21 Nov 2016 19:07:55 +0000"/>
                            <attachment id="144633" name="2016-11-22T16-37-51Z.tar.gz" size="1040377" author="jvf" created="Tue, 22 Nov 2016 18:21:46 +0000"/>
                            <attachment id="144401" name="cache-3-3-8.png" size="76023" author="kelsey.schubert@mongodb.com" created="Thu, 17 Nov 2016 22:31:06 +0000"/>
                            <attachment id="144400" name="cache-3-4-0-rc3.png" size="101141" author="kelsey.schubert@mongodb.com" created="Thu, 17 Nov 2016 22:31:07 +0000"/>
                            <attachment id="144257" name="diagnostic.data.338.tar.gz" size="912824" author="lilianr@us.ibm.com" created="Wed, 16 Nov 2016 02:17:19 +0000"/>
                            <attachment id="144229" name="diagnostic.data_3.4cr3.tar.gz" size="2584619" author="lilianr@us.ibm.com" created="Tue, 15 Nov 2016 21:40:14 +0000"/>
                            <attachment id="144303" name="stack.high.mongo" size="294626" author="lilianr@us.ibm.com" created="Wed, 16 Nov 2016 18:31:17 +0000"/>
                            <attachment id="144304" name="stack.low.mongo" size="302403" author="lilianr@us.ibm.com" created="Wed, 16 Nov 2016 18:31:04 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>13.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 15 Nov 2016 20:30:30 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        7 years, 11 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ian@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            7 years, 11 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>alexander.gorrod@mongodb.com</customfieldvalue>
            <customfieldvalue>dan@mongodb.com</customfieldvalue>
            <customfieldvalue>jvf</customfieldvalue>
            <customfieldvalue>lilianr@us.ibm.com</customfieldvalue>
            <customfieldvalue>michael.cahill@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrjqtj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hsrwun:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="1395">Storage 2016-12-12</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10750" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Steps To Reproduce</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>&lt;p&gt;How to reproduce:&lt;br/&gt;
use YCSB 6.0&lt;br/&gt;
Configuration:&lt;br/&gt;
fieldcount=10&lt;br/&gt;
fieldlength=10&lt;br/&gt;
recordcount=120000000&lt;br/&gt;
operationcount=100000000&lt;br/&gt;
workload=com.yahoo.ycsb.workloads.CoreWorkload&lt;br/&gt;
readallfields=true&lt;br/&gt;
readproportion=0.5&lt;br/&gt;
updateproportion=0.5&lt;br/&gt;
scanproportion=0&lt;br/&gt;
insertproportion=0&lt;/p&gt;

&lt;p&gt;System used: Power with 256GB of memory.&lt;/p&gt;</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[kelsey.schubert@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hs6y8f:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>