<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:43:57 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-17299] Extraneous memory allocations in snappy hurt WiredTiger performance on Windows</title>
                <link>https://jira.mongodb.org/browse/SERVER-17299</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;WiredTiger starts at an insertion rate of 40k/sec, and slows down to 15k/sec within 2 minutes.&lt;/p&gt;

&lt;p&gt;System memory is not exhausted.&lt;/p&gt;

&lt;p&gt;Specifically the CPU% appears to drop from 100% quite quickly and insert times (for the individual bulk inserts) grows.&lt;/p&gt;

&lt;p&gt;MongoStat doesn&apos;t appear to show any specific resource limit, nor do I see anything obvious.&lt;/p&gt;

&lt;p&gt;Using the C# driver 2.0.0-beta2.&lt;/p&gt;

&lt;p&gt;Please see &lt;a href=&quot;https://groups.google.com/forum/#!topic/mongodb-user/DMa24b2TqKM&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://groups.google.com/forum/#!topic/mongodb-user/DMa24b2TqKM&lt;/a&gt;&lt;/p&gt;</description>
                <environment></environment>
        <key id="184476">SERVER-17299</key>
            <summary>Extraneous memory allocations in snappy hurt WiredTiger performance on Windows</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="mark.benvenuto@mongodb.com">Mark Benvenuto</assignee>
                                    <reporter username="nickj">Nick Judson</reporter>
                        <labels>
                            <label>wiredtiger</label>
                    </labels>
                <created>Mon, 16 Feb 2015 16:52:27 +0000</created>
                <updated>Sat, 19 Sep 2015 00:16:11 +0000</updated>
                            <resolved>Tue, 24 Feb 2015 04:24:54 +0000</resolved>
                                    <version>3.0.0-rc8</version>
                                    <fixVersion>3.0.0-rc10</fixVersion>
                    <fixVersion>3.1.0</fixVersion>
                                    <component>WiredTiger</component>
                                        <votes>0</votes>
                                    <watches>14</watches>
                                                                                                                <comments>
                            <comment id="835731" author="xgen-internal-githook" created="Tue, 24 Feb 2015 12:40:02 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;michaelcahill&apos;, u&apos;name&apos;: u&apos;Michael Cahill&apos;, u&apos;email&apos;: u&apos;michael.cahill@wiredtiger.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17299&quot; title=&quot;Extraneous memory allocations in snappy hurt WiredTiger performance on Windows&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17299&quot;&gt;&lt;del&gt;SERVER-17299&lt;/del&gt;&lt;/a&gt; Eliminate memory allocation from snappy_compress.&lt;/p&gt;

&lt;p&gt;(cherry picked from commit c614b21cc563dbb400f7258124fc5d2ed58b4dce)&lt;br/&gt;
Branch: v3.0&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/96315ab194016300a663e61c8ba797d0416c23aa&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/96315ab194016300a663e61c8ba797d0416c23aa&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="835563" author="michael.cahill" created="Tue, 24 Feb 2015 04:24:54 +0000"  >&lt;p&gt;Fixed by changes to snappy to eliminate unnecessary memory allocation.&lt;/p&gt;</comment>
                            <comment id="835560" author="xgen-internal-githook" created="Tue, 24 Feb 2015 04:20:25 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;michaelcahill&apos;, u&apos;name&apos;: u&apos;Michael Cahill&apos;, u&apos;email&apos;: u&apos;michael.cahill@wiredtiger.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-17299&quot; title=&quot;Extraneous memory allocations in snappy hurt WiredTiger performance on Windows&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-17299&quot;&gt;&lt;del&gt;SERVER-17299&lt;/del&gt;&lt;/a&gt; Eliminate memory allocation from snappy_compress.&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/c614b21cc563dbb400f7258124fc5d2ed58b4dce&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/c614b21cc563dbb400f7258124fc5d2ed58b4dce&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="833242" author="nickj" created="Fri, 20 Feb 2015 00:59:34 +0000"  >&lt;p&gt;Great. Looking forward to kicking the tires on Snappy to see how it compares. zLib had very impressive compression.&lt;/p&gt;</comment>
                            <comment id="833238" author="dan@10gen.com" created="Fri, 20 Feb 2015 00:46:26 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=nick%40innsenroute.com&quot; class=&quot;user-hover&quot; rel=&quot;nick@innsenroute.com&quot;&gt;nick@innsenroute.com&lt;/a&gt;, as you can see from &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=mark.benvenuto&quot; class=&quot;user-hover&quot; rel=&quot;mark.benvenuto&quot;&gt;mark.benvenuto&lt;/a&gt;&apos;s comment above, we&apos;ve tracked this down to excess allocations in the snappy code.  We&apos;re working on a patch there.  In the meantime, thanks for your report and repro efforts.  Very helpful.  Please watch this ticket for updates soon.&lt;/p&gt;</comment>
                            <comment id="832979" author="nickj" created="Thu, 19 Feb 2015 20:07:26 +0000"  >&lt;p&gt;Setting the block compressor to snappy and the journal compressor to zlib still shows the slowdown, although not quite as dramatic as with both set to snappy.&lt;/p&gt;</comment>
                            <comment id="832906" author="nickj" created="Thu, 19 Feb 2015 18:53:01 +0000"  >&lt;p&gt;Switching from zlib to snappy causes the rapid slowdown I&apos;m seeing in my product. No compression also doesn&apos;t show the reduced cpu usage, but the performance is quite &apos;choppy&apos; even with very fast IO.&lt;/p&gt;</comment>
                            <comment id="832881" author="nickj" created="Thu, 19 Feb 2015 18:35:48 +0000"  >&lt;p&gt;My work box shows good, steady performance with zlib as well. As an aside, I notice that when compression is turned off for both block and journal, the memory usage will grow past the 1/2 system RAM.&lt;/p&gt;</comment>
                            <comment id="832739" author="nickj" created="Thu, 19 Feb 2015 16:20:56 +0000"  >&lt;p&gt;I tested with zlib instead of snappy on my home computer (3.1) it showed steady performance through the night. I&apos;ll re-run that test, and with compression off on my work dev box.&lt;/p&gt;</comment>
                            <comment id="832290" author="mark.benvenuto" created="Wed, 18 Feb 2015 23:07:57 +0000"  >&lt;p&gt;I have not been able to repro the exact same &quot;dive&quot; in performance, but I have seen a significant difference between none and snappy compression for the log (see &lt;b&gt;--wiredTigerJournalCompressor&lt;/b&gt;) with the attached repro.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sue.loverso&quot; class=&quot;user-hover&quot; rel=&quot;sue.loverso&quot;&gt;sue.loverso&lt;/a&gt;, while I gathered statistics, it was simply easier for me to attach the VS 2013 profiler, and examine where the bottleneck was. It turns out it was not in WT directly.&lt;/p&gt;

&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;Function Name&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;Module Name&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;Inclusive Samples&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;Exclusive Samples&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;Inclusive Samples %&lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt;Exclusive Samples %&lt;/th&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;mongo::WiredTigerRecoveryUnit::_commit&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;mongod.exe&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;70,800&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;171&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;65.27&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;0.16&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;mongo::WiredTigerRecoveryUnit::_txnClose&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;mongod.exe&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;70,179&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;103&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;64.69&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;0.09&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;__session_commit_transaction&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;mongod.exe&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;69,854&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;76&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;64.39&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;0.07&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;__wt_txn_commit&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;mongod.exe&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;69,771&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;61&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;64.32&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;0.06&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;__wt_log_write&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;mongod.exe&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;69,574&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;275&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;64.14&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;0.25&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;wt_snappy_compress&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;mongod.exe&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;61,390&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;77&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;56.59&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;0.07&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;snappy_compress&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;mongod.exe&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;61,313&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;36&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;56.52&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;0.03&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;snappy::RawCompress&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;mongod.exe&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;61,277&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;32&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;56.49&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;0.03&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;snappy::Compress&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;mongod.exe&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;61,237&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;131&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;56.45&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;0.12&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;-&amp;gt; &lt;b&gt;free&lt;/b&gt;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;mongod.exe&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;28,827&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;68&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;26.57&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;0.06&lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;-&amp;gt; &lt;b&gt;operator new&lt;/b&gt;&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;mongod.exe&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;28,479&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;40&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;26.25&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;0.04&lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;


&lt;p&gt;From &lt;tt&gt;Compress&lt;/tt&gt; in &lt;tt&gt;snappy.cc&lt;/tt&gt;, we see that max_output is constantly being allocated and deallocated.&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    // Compress input_fragment and append to dest&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    const int max_output = MaxCompressedLength(num_to_read);&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    // Need a scratch buffer for the output, in case the byte sink doesn&apos;t&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    // have room for us directly.&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    if (scratch_output == NULL) {&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      scratch_output = new char[max_output];&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    } else {&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      // Since we encode kBlockSize regions followed by a region&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      // which is &amp;lt;= kBlockSize in length, a previously allocated&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      // scratch_output[] region is big enough for this iteration.&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;This leaves a few possible ways to improve this:&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;Call &lt;tt&gt;snappy_compress&lt;/tt&gt; less often by having fewer threads compress the journal&lt;/li&gt;
	&lt;li&gt;Change snappy to take scratch buffers from WT instead of allocating them itself.&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;The max size of max_output is &lt;tt&gt;MaxCompressedLength(num_to_read = min(N, kBlockSize))&lt;/tt&gt; where &lt;tt&gt;N&lt;/tt&gt; = the size of the input, and &lt;tt&gt;kBlockSize&lt;/tt&gt; is 65536. We just need to pass two buffers, one for scratch (which I did not show, and has a max of 65336), and one of for scratch_output which has a max size outlined below.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sue.loverso&quot; class=&quot;user-hover&quot; rel=&quot;sue.loverso&quot;&gt;sue.loverso&lt;/a&gt; What are suggestions on how to improve this?&lt;/p&gt;</comment>
                            <comment id="831393" author="dan@10gen.com" created="Wed, 18 Feb 2015 05:08:32 +0000"  >&lt;p&gt;Thanks for attaching this Nick.  We&apos;re taking a look.&lt;/p&gt;</comment>
                            <comment id="831271" author="nickj" created="Wed, 18 Feb 2015 01:06:14 +0000"  >&lt;p&gt;mongo 8: Same application under MMap shows steady state performance.&lt;/p&gt;</comment>
                            <comment id="831238" author="nickj" created="Wed, 18 Feb 2015 00:11:32 +0000"  >&lt;p&gt;mongo4.png - mongo7.png: Time sequence showing slow-down / reduced cpu usage in my product. This is write-once and read-once.&lt;/p&gt;</comment>
                            <comment id="831211" author="nickj" created="Tue, 17 Feb 2015 23:58:48 +0000"  >&lt;p&gt;mongo3: After 4 mintes, rc8&lt;/p&gt;</comment>
                            <comment id="831209" author="nickj" created="Tue, 17 Feb 2015 23:58:22 +0000"  >&lt;p&gt;mongo2: Initial speeds with rc8.&lt;/p&gt;</comment>
                            <comment id="831206" author="nickj" created="Tue, 17 Feb 2015 23:51:27 +0000"  >&lt;p&gt;mongo 1.png: Latest daily build (3.1, 2/17/15) shows a sudden drop in speed and cpu usage.&lt;/p&gt;</comment>
                            <comment id="831204" author="nickj" created="Tue, 17 Feb 2015 23:50:06 +0000"  >&lt;p&gt;Sample app showing slowdown.&lt;/p&gt;</comment>
                            <comment id="830172" author="dan@10gen.com" created="Mon, 16 Feb 2015 23:17:43 +0000"  >&lt;p&gt;I think long run, it will probably be better to have a reproducer in hand.&lt;br/&gt;
If it&apos;s too much trouble, lmk.&lt;/p&gt;

&lt;p&gt;Thanks for the report.&lt;/p&gt;

&lt;p&gt;On Mon, Feb 16, 2015 at 6:14 PM, Nick Judson (JIRA) &amp;lt;jira@mongodb.org&amp;gt;&lt;/p&gt;
</comment>
                            <comment id="830167" author="nickj" created="Mon, 16 Feb 2015 23:13:07 +0000"  >&lt;p&gt;Ah - I see you want me to attach it to this ticket. It&apos;s part of a larger project however I&apos;ll attempt to split it out and get something over to you.&lt;/p&gt;

&lt;p&gt;Alternately you could take over my dev machine.&lt;/p&gt;</comment>
                            <comment id="830162" author="nickj" created="Mon, 16 Feb 2015 23:06:41 +0000"  >&lt;p&gt;Sure. How do you want to look at this?&lt;/p&gt;</comment>
                            <comment id="830156" author="dan@10gen.com" created="Mon, 16 Feb 2015 22:56:07 +0000"  >&lt;p&gt;Hi Nick, Is it possible to attach your workload generator?  &lt;/p&gt;</comment>
                            <comment id="829989" author="nickj" created="Mon, 16 Feb 2015 16:55:29 +0000"  >&lt;p&gt;&lt;a href=&quot;https://mms.mongodb.com/host/detail/54299fbce4b0b14ef1439821/fed5100f74ed5ea839bb8118054296c0&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://mms.mongodb.com/host/detail/54299fbce4b0b14ef1439821/fed5100f74ed5ea839bb8118054296c0&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="829988" author="nickj" created="Mon, 16 Feb 2015 16:54:30 +0000"  >&lt;p&gt;Windows 8.1, 64bit, 16GB RAM, All SSD, i7-5760X.&lt;/p&gt;</comment>
                            <comment id="829987" author="nickj" created="Mon, 16 Feb 2015 16:53:33 +0000"  >&lt;p&gt;Let me know if you want me to move all the information from the forum into jira.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                            <outwardlinks description="duplicates">
                                                        </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="65021" name="MongoPerfTest.7z" size="9198388" author="nickj" created="Tue, 17 Feb 2015 23:50:06 +0000"/>
                            <attachment id="65022" name="mongo1.png" size="308829" author="nickj" created="Tue, 17 Feb 2015 23:51:27 +0000"/>
                            <attachment id="65023" name="mongo2.png" size="390148" author="nickj" created="Tue, 17 Feb 2015 23:58:22 +0000"/>
                            <attachment id="65024" name="mongo3.png" size="384050" author="nickj" created="Tue, 17 Feb 2015 23:58:48 +0000"/>
                            <attachment id="65029" name="mongo4.png" size="1102135" author="nickj" created="Wed, 18 Feb 2015 00:11:32 +0000"/>
                            <attachment id="65028" name="mongo5.png" size="1097453" author="nickj" created="Wed, 18 Feb 2015 00:11:32 +0000"/>
                            <attachment id="65027" name="mongo6.png" size="1106614" author="nickj" created="Wed, 18 Feb 2015 00:11:32 +0000"/>
                            <attachment id="65026" name="mongo7.png" size="1105646" author="nickj" created="Wed, 18 Feb 2015 00:11:32 +0000"/>
                            <attachment id="65035" name="mongo8.png" size="851943" author="nickj" created="Wed, 18 Feb 2015 01:06:14 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>24.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_12451" key="com.atlassian.jira.plugin.system.customfieldtypes:multiversion">
                        <customfieldname>Backport Completed</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="14850">3.0.0-rc10</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Mon, 16 Feb 2015 22:56:07 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        8 years, 51 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            8 years, 51 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>dan@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>mark.benvenuto@mongodb.com</customfieldvalue>
            <customfieldvalue>michael.cahill@mongodb.com</customfieldvalue>
            <customfieldvalue>nickj</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrlcbr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hs6c2v:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>162225</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10750" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Steps To Reproduce</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>&lt;p&gt;I have 50 threads bulk inserting 100 &amp;lt;1k documents as fast as Mongo will take them.&lt;/p&gt;

&lt;p&gt;Using the MMap engine doesn&apos;t show this behavior. Right now, after running for ~8 minutes, my cpu is sitting at 23% and the insert rate is at 11k/s.&lt;/p&gt;

&lt;p&gt;I don&apos;t know if this is expected or not, but it seems like quite a dramatic drop off.&lt;/p&gt;</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrst9b:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>