<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:27:54 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-31694] 17% throughput regression in insert workload</title>
                <link>https://jira.mongodb.org/browse/SERVER-31694</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Simple insert workload:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    res = benchRun({&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        ops: [&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;            {op: &quot;insert&quot;, doc: {},  ns: &quot;test.c&quot;},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        ],&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        seconds: 45,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        parallel: 15&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    })&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    printjson(res)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;standalone shows ~11% throughput reduction&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;3.4.9:     &quot;totalOps/s&quot; : 312257.31243007537,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;3.6.0-rc0  &quot;totalOps/s&quot; : 277214.2306740861,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;1-node replica set shows ~50% throughput reduction&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;3.4.9:     &quot;totalOps/s&quot; : 181808.21357151057,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;3.6.0-rc0  &quot;totalOps/s&quot; : 91202.31599375796,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Metrics for the 1-node replica set case suggest issue may be related to large number of write-lock calls, about 3 per insert:&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/169119/169119_run2.png&quot; width=&quot;100%&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;</description>
                <environment></environment>
        <key id="449967">SERVER-31694</key>
            <summary>17% throughput regression in insert workload</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="4">Incomplete</resolution>
                                        <assignee username="backlog-server-repl">Backlog - Replication Team</assignee>
                                    <reporter username="bruce.lucas@mongodb.com">Bruce Lucas</reporter>
                        <labels>
                    </labels>
                <created>Tue, 24 Oct 2017 15:04:43 +0000</created>
                <updated>Tue, 6 Dec 2022 03:48:32 +0000</updated>
                            <resolved>Fri, 28 Oct 2022 14:10:20 +0000</resolved>
                                    <version>3.6.0</version>
                                                    <component>Storage</component>
                    <component>Write Ops</component>
                                        <votes>3</votes>
                                    <watches>45</watches>
                                                                                                                <comments>
                            <comment id="4935482" author="judah.schvimer" created="Fri, 28 Oct 2022 14:10:20 +0000"  >&lt;p&gt;This regression is between two EOL versions of MongoDB, and the Server has diverged significantly since this occurred. Bulk insert performance continues to be something we track and aim to improve over time. Future performance improvements will be made using new analyses on active versions of the Server. Closing this as &quot;Incomplete&quot;.&lt;/p&gt;</comment>
                            <comment id="2186824" author="schwerin" created="Wed, 20 Mar 2019 23:21:52 +0000"  >&lt;p&gt;Great analysis, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bartle&quot; class=&quot;user-hover&quot; rel=&quot;bartle&quot;&gt;bartle&lt;/a&gt;. &lt;tt&gt;setMyLastAppliedOpTimeForward&lt;/tt&gt; and more generally the replication coordinator mutex are known contention points in the system, today. That said, I don&apos;t see a ticket specifically tracking it, and I think this ticket actually covers a separate case involving w:1 bulk insert performance. I have filed &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-40250&quot; title=&quot;High contention for ReplicationCoordinatorImpl::_mutex in w:majority workloads&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-40250&quot;&gt;&lt;del&gt;SERVER-40250&lt;/del&gt;&lt;/a&gt; to track the issue you report in your comment above.&lt;/p&gt;

&lt;p&gt;CC &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=alyson.cabral&quot; class=&quot;user-hover&quot; rel=&quot;alyson.cabral&quot;&gt;alyson.cabral&lt;/a&gt; and &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sheeri.cabral&quot; class=&quot;user-hover&quot; rel=&quot;sheeri.cabral&quot;&gt;sheeri.cabral&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="2185867" author="bartle" created="Wed, 20 Mar 2019 05:26:47 +0000"  >&lt;p&gt;Are there plans to improve performance of&#160;&lt;tt&gt;setMyLastAppliedOpTimeForward&lt;/tt&gt;?&#160; On a write-majority, insert-heavy workflow we basically see single-core contention on &lt;tt&gt;setMyLastAppliedOpTimeForward&lt;/tt&gt; (based on a CPU profile).&#160; That particular function takes an exclusive mutex, so it&apos;s unsurprising that if you push enough write-majority writes through you&apos;d contend on a single core (in practice we&apos;re hitting a bottleneck of 12k wps, on a 16-core machine, with ~7% CPU usage).&lt;/p&gt;

&lt;p&gt;Ultimately. all of the CPU ends up in &lt;tt&gt;_wakeReadyWaiters_inlock&lt;/tt&gt;.&#160; That particular implementation seems rather naive; it ends up recomputing a bunch of things (again, under a global, exclusive lock) for every replication waiter.&#160; Instead, it seems like you should structure this code such that it determines the largest optime that satisfies the various write concern modes (basically &quot;majority&quot; and w=&quot;N&quot;) once, and then pass that information down into &lt;tt&gt;_doneWaitingForReplication_inlock&lt;/tt&gt;.&lt;/p&gt;

&lt;p&gt;Beyond this, reading through the code, it&apos;s fairly concerning how coarse-grained &lt;tt&gt;_mutex&lt;/tt&gt; on &lt;tt&gt;ReplicationCoordinatorImpl&lt;/tt&gt; is.&#160; Is there a reason more work hasn&apos;t been invested in finer-grained locks, or even reader-writer locks?&#160; As-is, it&apos;s really difficult to make any perf improvements.&lt;/p&gt;</comment>
                            <comment id="1915721" author="victorgp" created="Fri, 8 Jun 2018 20:51:27 +0000"  >&lt;p&gt;Yes, this is our case, i guess upserts are also affected. We have a heavy write pattern so bulk inserts are needed.&lt;/p&gt;

&lt;p&gt;Isn&apos;t this usage of bulk inserts a common operation among MongoDB users? I hardly believe most users will use simple inserts when they have the option of doing them bulked.&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;</comment>
                            <comment id="1915086" author="schwerin" created="Fri, 8 Jun 2018 13:46:08 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=victorgp&quot; class=&quot;user-hover&quot; rel=&quot;victorgp&quot;&gt;victorgp&lt;/a&gt; and &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=stutiredboy%40gmail.com&quot; class=&quot;user-hover&quot; rel=&quot;stutiredboy@gmail.com&quot;&gt;stutiredboy@gmail.com&lt;/a&gt;, this ticket tracks a very particular performance regression when clients are using the &lt;a href=&quot;https://docs.mongodb.com/manual/reference/method/Bulk.insert/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;Bulk.insert&lt;/a&gt; or related driver functionality to efficiently insert a large volume of documents. Is this the action you&apos;re performing when you experience your performance problem? If not, I would suggest filing another ticket.&lt;/p&gt;

&lt;p&gt;As for this particular performance regression, we hope to address it this year.&lt;/p&gt;</comment>
                            <comment id="1913308" author="stutiredboy@gmail.com" created="Thu, 7 Jun 2018 02:00:35 +0000"  >&lt;p&gt;Seconded.&lt;/p&gt;

&lt;p&gt;The declining performance is a big problem for us,&lt;/p&gt;

&lt;p&gt;We have already decided to give up the 3.6.x version but staying on 3.4.&lt;/p&gt;</comment>
                            <comment id="1912650" author="victorgp" created="Wed, 6 Jun 2018 17:21:32 +0000"  >&lt;p&gt;Can we get an update on how this is going and when is it going to be released? Such a performance impact in a new MongoDB version should be a big concern. This was reported in October 2017, it has Major priority but after ~8months there is not a fix yet, and that is a bit concerning.&lt;/p&gt;

&lt;p&gt;In our case, we can&apos;t upgrade our infrastructure to 3.6, the performance degradation is too big for an optimal database.&lt;/p&gt;

&lt;p&gt;Also, last comment is from 1st Feb, hope this was not forgotten...&lt;/p&gt;</comment>
                            <comment id="1791982" author="schwerin" created="Thu, 1 Feb 2018 17:03:22 +0000"  >&lt;p&gt;Moving to &quot;Needs Triage&quot; so replication team can assess when/how to schedule this. Given that it was in 3.7 Required, I expect it&apos;ll be going back to 3.7 Required as part of the re-triage.&lt;/p&gt;</comment>
                            <comment id="1791764" author="sue.loverso" created="Thu, 1 Feb 2018 15:14:40 +0000"  >&lt;p&gt;I have made changes in WiredTiger develop to alleviate blocking on the timestamp queue locks. In WiredTiger-only timestamp tests I see 12% improvement. When I run that against this workload I see no improvement. Pmp is still showing a lot of contention and I frequently see the thread holding the lock in the &lt;tt&gt;now&lt;/tt&gt; call, presumably &lt;tt&gt;gettimeofday&lt;/tt&gt;. I don&apos;t believe WiredTiger is the underlying cause and reducing the lock contention in &lt;tt&gt;repl::getNextOpTimes&lt;/tt&gt; and &lt;tt&gt;ReplicationCoordinatorImpl::setMyLastAppliedOpTimeForward&lt;/tt&gt; seems to be the next step.&lt;/p&gt;</comment>
                            <comment id="1778079" author="sue.loverso" created="Thu, 18 Jan 2018 18:29:23 +0000"  >&lt;p&gt;Running the mongodb 3.6 branch against the WiredTiger 3.8 branch shows a small, but very consistent 3% degradation in the insert workload (about 96.7k) compared to using the WiredTiger 3.6 branch.  I am going to do some bisecting to see if it is a single change that accounts for it.&lt;/p&gt;

&lt;p&gt;ETA: The change that causes the degradation was the addition of lock tracking statistics around the three rwlocks in WiredTiger in the transaction subsystem: the global rwlock, and the two rwlocks around the two timestamp queues. Those changes were added in &lt;a href=&quot;https://jira.mongodb.org/browse/WT-3725&quot; title=&quot;Add statistics to locks around timestamp structures&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-3725&quot;&gt;&lt;del&gt;WT-3725&lt;/del&gt;&lt;/a&gt;. I have opened &lt;a href=&quot;https://jira.mongodb.org/browse/WT-3857&quot; title=&quot;Timestamp lock tracking statistics hurt performance&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-3857&quot;&gt;&lt;del&gt;WT-3857&lt;/del&gt;&lt;/a&gt; to consider whether those should be removed or modified in some way.&lt;/p&gt;

&lt;p&gt;If I turn off tracking on those three locks, then the WiredTiger 3.8 branch is on par with the WiredTiger 3.6 branch when run with the MongoDB 3.6 branch.&lt;/p&gt;</comment>
                            <comment id="1776976" author="sue.loverso" created="Wed, 17 Jan 2018 19:01:20 +0000"  >&lt;p&gt;Since this is back on my plate I did baseline runs to see what the current situation is. Given above discussion, I ran on 3.4 with majority read concern turned on. Here are my results (avg of 3 runs):&lt;/p&gt;
&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;th class=&apos;confluenceTh&apos;&gt; Branch &lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt; Changeset &lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt; Insert ops/sec &lt;/th&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 3.4.9 RC &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; ce08a408 &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 113.3k &lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 3.6 &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 85cf50d91d &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 99.4k &lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;


&lt;p&gt;So the 3.6 performance has not changed since my last runs back in November. I ran pmp on several runs to see if any patterns emerge from the stacks. I continue to see lots of contention in &lt;tt&gt;setMyLastAppliedOpTimeForward&lt;/tt&gt; and &lt;tt&gt;getNextOpTime&lt;/tt&gt;. In particular, in every run I see the thread holding the lock from &lt;tt&gt;setMyLastAppliedOpTimeForward&lt;/tt&gt; in &lt;tt&gt;&amp;#95;replExecutor-&amp;gt;now()&lt;/tt&gt;. Here&apos;s a good illustration, as the test is using insert 15 threads and here are 14 of them:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;     13 __lll_lock_wait,_L_lock_791,pthread_mutex_lock,__gthread_mutex_lock,lock,lock,unique_lock,mongo::repl::ReplicationCoordinatorImpl::setMyLastAppliedOpTimeForward,operator(),mongo::RecoveryUnit::OnCommitChange::commit,mongo::repl::_logOpsInner,mongo::WiredTigerRecoveryUnit::_commit,commit,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,operator()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      1 mongo::executor::ThreadPoolTaskExecutor::now,mongo::repl::ReplicationCoordinatorImpl::_setMyLastAppliedOpTime_inlock,mongo::repl::ReplicationCoordinatorImpl::setMyLastAppliedOpTimeForward,operator(),mongo::RecoveryUnit::OnCommitChange::commit,mongo::repl::_logOpsInner,mongo::WiredTigerRecoveryUnit::_commit,commit,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,operator()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;In another run, I also see some contention on the WiredTiger commit timestamp queue write lock. Eight threads are waiting on the mutex in &lt;tt&gt;getNextOpTimes&lt;/tt&gt; and the thread holding it has the commit timestamp queue lock but other threads in WiredTiger are blocked on that lock:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      8 __lll_lock_wait,_L_lock_791,pthread_mutex_lock,__gthread_mutex_lock,lock,lock_guard,mongo::repl::(anonymous,mongo::repl::getNextOpTimes,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,operator()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      1 __wt_writeunlock,__wt_txn_set_commit_timestamp,__wt_txn_set_timestamp,__session_timestamp_transaction,mongo::WiredTigerRecoveryUnit::setTimestamp,mongo::WiredTigerRecordStore::oplogDiskLocRegister,mongo::repl::(anonymous,mongo::repl::getNextOpTimes,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,operator()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      1 sched_yield,__wt_yield,__wt_writelock,__wt_txn_clear_commit_timestamp,__wt_txn_release,__wt_txn_commit,__session_commit_transaction,mongo::WiredTigerRecoveryUnit::_txnClose,mongo::WiredTigerRecoveryUnit::_commit,commit,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,operator()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      1 pthread_cond_timedwait@@GLIBC_2.3.2,__wt_cond_wait_signal,__wt_cond_wait,__wt_writelock,__wt_txn_clear_commit_timestamp,__wt_txn_release,__wt_txn_commit,__session_commit_transaction,mongo::WiredTigerRecoveryUnit::_txnClose,mongo::WiredTigerRecoveryUnit::_commit,commit,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,operator()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="1748628" author="bruce.lucas@10gen.com" created="Mon, 11 Dec 2017 17:37:15 +0000"  >&lt;p&gt;Situation on a 2-node repl set is similar:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;3.4.10 default settings             &quot;totalOps/s&quot; : 162961.08065147756,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;3.4.10 --enableMajorityReadConcern  &quot;totalOps/s&quot; : 149420.53540043414,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;3.6.0  default settings             &quot;totalOps/s&quot; : 131686.83374001557,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="1748549" author="bruce.lucas@10gen.com" created="Mon, 11 Dec 2017 17:00:30 +0000"  >&lt;p&gt;In the test on this ticket enabling majority read concern in 3.4.10 does decrease performance some, by about 5%, but not by the 17% decrease seen at the default settings in 3.6.0.&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;3.4.10 --enableMajorityReadConcern  &quot;totalOps/s&quot; : 174735.2184719974,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="1748368" author="geert.bosch" created="Mon, 11 Dec 2017 14:52:41 +0000"  >&lt;p&gt;The picture changes a bit when you compare with 3.4 with majority reads enabled. IIRC, we&apos;re about at parity there. Of course, we now have enabled something by default that users may care less about than 17% perf difference.&lt;/p&gt;</comment>
                            <comment id="1748321" author="bruce.lucas@10gen.com" created="Mon, 11 Dec 2017 14:21:23 +0000"  >&lt;p&gt;Status update: 3.6.0 is much improved compared to the test on 3.6.0-rc0 in the original description, but still shows ~17% performance regression compared to 3.4.10:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;3.4.10:    &quot;totalOps/s&quot; : 183103.84044700526,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;3.6.0-rc0  &quot;totalOps/s&quot; : 91202.31599375796,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;3.6.0:     &quot;totalOps/s&quot; : 151687.90105520733,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="1736850" author="michael.cahill" created="Tue, 28 Nov 2017 23:16:31 +0000"  >&lt;p&gt;Reopening &amp;#8211; since this workload isn&apos;t automated (yet), we need to keep this ticket open to make sure we drive down the remaining regression.&lt;/p&gt;</comment>
                            <comment id="1736168" author="milkie" created="Tue, 28 Nov 2017 16:31:42 +0000"  >&lt;p&gt;I&apos;m closing this ticket now, as we have no remaining immediate server changes planned to improve this general performance issue.  We will continue working on the individual linked BF tickets, and such work may spawn new SERVER tickets if the work is to be done in the server.&lt;/p&gt;</comment>
                            <comment id="1735187" author="sue.loverso" created="Mon, 27 Nov 2017 17:40:40 +0000"  >&lt;p&gt;I have run with the current mongodb master as of this morning (61122064) and WT develop (00374e0c7) and I&apos;m seeing a solid 10%+ improvement from last week. These results are from a single-node replica set:&lt;/p&gt;
&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;th class=&apos;confluenceTh&apos;&gt; Branch &lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt; insert &lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt; TTL test &lt;/th&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 3.4.9 &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 126k &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 255k &lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; Nov20 &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 89k &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 125k &lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; Nov27 &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 99k &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 140k &lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;


&lt;p&gt;Running pmp (twice) on the current code during the insert test now shows consistently about 5 insert threads waiting on the mutex in &lt;tt&gt;getNextOpTimes&lt;/tt&gt;, 1 or 2 other insert threads waiting on the ReplicationCoordinator mutex and all the other threads doing real work.  That is a big improvement!&lt;/p&gt;
</comment>
                            <comment id="1731955" author="milkie" created="Tue, 21 Nov 2017 18:50:22 +0000"  >&lt;p&gt;I&apos;ve been working on making the calls to _setStableTimestampForStorage_inlock() more efficient, on the primary write path, but am seeing mixed results.  In some cases it performs better and in some cases, especially when there is cache pressure, the performance is much worse.  I&apos;m continuing to investigate.&lt;/p&gt;</comment>
                            <comment id="1731596" author="sue.loverso" created="Tue, 21 Nov 2017 15:12:54 +0000"  >&lt;p&gt;That is correct.&lt;/p&gt;</comment>
                            <comment id="1731290" author="michael.cahill" created="Tue, 21 Nov 2017 05:35:49 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sue.loverso&quot; class=&quot;user-hover&quot; rel=&quot;sue.loverso&quot;&gt;sue.loverso&lt;/a&gt;, just to make sure we&apos;re on the same page: MongoDB 3.6 is not setting WiredTiger&apos;s &lt;tt&gt;stable_timestamp&lt;/tt&gt; at all &amp;#8211; all checkpoints write with a current transaction snapshot and no read timestamp.&lt;/p&gt;</comment>
                            <comment id="1730287" author="sue.loverso" created="Mon, 20 Nov 2017 15:02:43 +0000"  >&lt;blockquote&gt;&lt;p&gt;If I understand correctly, the stable timestamp is not meaningful when we&apos;re primary,&lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Internally to WiredTiger the stable timestamp is meaningful on every machine as it is a visibility mechanism for checkpoints and eviction. So it must be updated frequently enough on a primary as well.&lt;/p&gt;</comment>
                            <comment id="1729446" author="geert.bosch" created="Fri, 17 Nov 2017 23:15:31 +0000"  >&lt;p&gt;There are various slowdowns, but the one that stands out is &lt;tt&gt;ReplicationCoordinatorImpl::_setStableTimestampForStorage_inlock&lt;/tt&gt;, as it is quite expensive (about 20% as expensive as &lt;tt&gt;_insertRecords&lt;/tt&gt;, which inserts and indexes the actual data), but executes inside of the ReplicationCoordinator mutex.&lt;/p&gt;

&lt;p&gt;If I understand correctly, the stable timestamp is not meaningful when we&apos;re primary, as any majority confirmed timestamp is stable. When we&apos;re secondary, we only try to advance the stable timestamp once per batch, so it is not performance critical.&lt;/p&gt;</comment>
                            <comment id="1724403" author="sue.loverso" created="Mon, 13 Nov 2017 19:27:07 +0000"  >&lt;p&gt;I reran with the latest changes in mongodb master from several tickets and the latest drop of WiredTiger.  Running changeset &lt;tt&gt;9fbb13033&lt;/tt&gt; and using the internal wiredtiger code I see the same performance gains that we saw with the hacked changes, about 91K ops/sec.&lt;/p&gt;

&lt;p&gt;Running pmp shows 13 of 15 workload threads in one stack:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;     13 __lll_lock_wait,_L_lock_791,pthread_mutex_lock,__gthread_mutex_lock,lock,lock,unique_lock,mongo::repl::ReplicationCoordinatorImpl::setMyLastAppliedOpTimeForward,operator(),mongo::RecoveryUnit::OnCommitChange::commit,mongo::repl::_logOpsInner,mongo::WiredTigerRecoveryUnit::_commit,commit,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,operator()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="1722478" author="milkie" created="Fri, 10 Nov 2017 13:44:06 +0000"  >&lt;p&gt;I linked two more tickets to help improve performance.  After these tickets go in, we should reevaluate the throughput and then decide if we need to do more.  With the improvements made thus far, I don&apos;t think this needs to block the next rc.&lt;/p&gt;</comment>
                            <comment id="1722306" author="michael.cahill" created="Fri, 10 Nov 2017 05:22:41 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=milkie&quot; class=&quot;user-hover&quot; rel=&quot;milkie&quot;&gt;milkie&lt;/a&gt;, is there anything to do in this ticket?  Now that we have &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-31899&quot; title=&quot;Improve getTerm() performance&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-31899&quot;&gt;&lt;del&gt;SERVER-31899&lt;/del&gt;&lt;/a&gt;, can we close this as a dup?&lt;/p&gt;</comment>
                            <comment id="1722054" author="milkie" created="Thu, 9 Nov 2017 21:29:32 +0000"  >&lt;p&gt;I filed &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-31899&quot; title=&quot;Improve getTerm() performance&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-31899&quot;&gt;&lt;del&gt;SERVER-31899&lt;/del&gt;&lt;/a&gt; to try to get back more performance &amp;#8211; this is the mutex that shows that calls to getTerm() and setMyLastAppliedOptimeForward() were hitting contention.&lt;/p&gt;</comment>
                            <comment id="1721446" author="milkie" created="Thu, 9 Nov 2017 14:31:01 +0000"  >&lt;p&gt;I have now pushed two commits into the master branch that should have a positive effect on performance of this.  Continuing to look at perf numbers for the next bottleneck to address.&lt;/p&gt;</comment>
                            <comment id="1720485" author="sue.loverso" created="Wed, 8 Nov 2017 16:30:14 +0000"  >&lt;p&gt; I have attached the &lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/attachment/170505/170505_pmp.patch.sesscache&quot; title=&quot;pmp.patch.sesscache attached to SERVER-31694&quot;&gt;full pmp output &lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.mongodb.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt; for the patched run with contention on the SessionCache mutex.  I do not see an obvious thread that holds it.  Note that no threads are active in WiredTiger.  &lt;/p&gt;</comment>
                            <comment id="1720410" author="sue.loverso" created="Wed, 8 Nov 2017 15:40:16 +0000"  >&lt;p&gt;The second run shows very different stacks.  I see a lot of contention on the SessionCache mutex including threads in Michael&apos;s patch &lt;tt&gt;ensureSession&lt;/tt&gt;:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      7 __lll_lock_wait,_L_lock_791,pthread_mutex_lock,__gthread_mutex_lock,lock,lock_guard,mongo::WiredTigerSessionCache::releaseSession,mongo::WiredTigerRecoveryUnit::~WiredTigerRecoveryUnit,operator(),~unique_ptr,~OperationContext,~OperationContext,mongo::ServiceContext::OperationContextDeleter::operator(),reset,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,operator()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      3 __lll_lock_wait,_L_lock_791,pthread_mutex_lock,__gthread_mutex_lock,lock,lock_guard,mongo::WiredTigerSessionCache::getSession,mongo::WiredTigerRecoveryUnit::_ensureSession,_ensureSession,mongo::WiredTigerRecoveryUnit::_txnOpen,mongo::repl::(anonymous,mongo::repl::getNextOpTimes,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,operator()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      1 pthread_mutex_lock,__gthread_mutex_lock,lock,lock_guard,mongo::WiredTigerSessionCache::getSession,mongo::WiredTigerRecoveryUnit::_ensureSession,_ensureSession,mongo::WiredTigerRecoveryUnit::_txnOpen,mongo::repl::(anonymous,mongo::repl::getNextOpTimes,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,operator()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      1 __lll_lock_wait,_L_lock_791,pthread_mutex_lock,__gthread_mutex_lock,lock,lock_guard,mongo::WiredTigerSessionCache::releaseSession,operator(),~unique_ptr,mongo::WiredTigerSessionCache::waitUntilDurable&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;There are also two threads waiting on the repl coordinator mutex in &lt;tt&gt;setMyLastAppliedOpTimeForward&lt;/tt&gt; and one waiting on the newOpMutex in &lt;tt&gt;getNextOpTimes&lt;/tt&gt;.&lt;/p&gt;</comment>
                            <comment id="1720402" author="sue.loverso" created="Wed, 8 Nov 2017 15:32:12 +0000"  >&lt;p&gt;This is very interesting.  I ran pmp on two separate runs of the test and saw very different output.  On one run, I still see more than half the workload threads waiting on the repl coordinator mutex in &lt;tt&gt;setMyLastAppliedOpTimeForward&lt;/tt&gt; even with the small change Michael added.  Also it would make sense for the same change to go into &lt;tt&gt;setMyLastDurableOpTimeForward&lt;/tt&gt;:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      7 __lll_lock_wait,_L_lock_791,pthread_mutex_lock,__gthread_mutex_lock,lock,std::unique_lock&amp;lt;std::mutex&amp;gt;::lock,unique_lock,mongo::repl::ReplicationCoordinatorImpl::setMyLastAppliedOpTimeForward,operator(),mongo::RecoveryUnit::OnCommitChange::commit,mongo::repl::_logOpsInner,mongo::WiredTigerRecoveryUnit::_commit,commit,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,operator()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      1 __lll_unlock_wake,_L_unlock_697,pthread_mutex_unlock,__gthread_mutex_unlock,unlock,std::unique_lock&amp;lt;std::mutex&amp;gt;::unlock,~unique_lock,mongo::repl::ReplicationCoordinatorImpl::setMyLastAppliedOpTimeForward,operator(),mongo::RecoveryUnit::OnCommitChange::commit,mongo::repl::_logOpsInner,mongo::WiredTigerRecoveryUnit::_commit,commit,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,operator()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      1 __lll_lock_wait,_L_lock_791,pthread_mutex_lock,__gthread_mutex_lock,lock,lock,unique_lock,mongo::repl::ReplicationCoordinatorImpl::setMyLastDurableOpTimeForward,mongo::WiredTigerSessionCache::waitUntilDurable&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;The thread holding it is at the beginning of the worker function:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      1 mongo::executor::ThreadPoolTaskExecutor::now,mongo::repl::ReplicationCoordinatorImpl::_setMyLastAppliedOpTime_inlock,mongo::repl::ReplicationCoordinatorImpl::setMyLastAppliedOpTimeForward,operator(),mongo::RecoveryUnit::OnCommitChange::commit,mongo::repl::_logOpsInner,mongo::WiredTigerRecoveryUnit::_commit,commit,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,operator()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="1720377" author="sue.loverso" created="Wed, 8 Nov 2017 15:08:49 +0000"  >&lt;p&gt;Interestingly, Michael and I both ran with his patch installed and saw different stacks.  He frequently saw threads in &lt;tt&gt;synchronizeOnOplogInFlightResource&lt;/tt&gt; and my runs saw &lt;tt&gt;setMyLastAppliedOpTimeForward&lt;/tt&gt; and &lt;tt&gt;getTerm&lt;/tt&gt;.  I am running on AWS, I don&apos;t know where Michael is running.  These numbers are all with a production build of WiredTiger (my earlier numbers were with a diagnostic build):&lt;/p&gt;

&lt;p&gt;Here are my results today:&lt;/p&gt;
&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;th class=&apos;confluenceTh&apos;&gt; MongoDB Build &lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt; WiredTiger Build &lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt; ops/sec &lt;/th&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; r3.5.8 &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; mongodb-3.5.8 &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 113k &lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; master (11e68f0) &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; develop (d51a7e37) &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 69k &lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; master+all diffs above&lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; &lt;a href=&quot;https://jira.mongodb.org/browse/WT-3715&quot; title=&quot;Performance tuning for cache overflow mechanism&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-3715&quot;&gt;&lt;del&gt;WT-3715&lt;/del&gt;&lt;/a&gt; branch &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 90k &lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;


&lt;p&gt;With those changes we&apos;ve regained a large portion of the performance, but we&apos;re still below 3.5.8.  I&apos;ll keep looking into the stacks of the patched runs and see if I can find more bottlenecks.&lt;/p&gt;</comment>
                            <comment id="1720015" author="milkie" created="Wed, 8 Nov 2017 02:46:49 +0000"  >&lt;p&gt;Nice find. I agree that&#8217;s a good change to make. Tomorrow morning I&#8217;ll take Michael&#8217;s patch and prepare it for commit. &lt;/p&gt;</comment>
                            <comment id="1719918" author="sue.loverso" created="Tue, 7 Nov 2017 23:12:01 +0000"  >&lt;p&gt;While I focused yesterday on WiredTiger locking statistics I think the biggest problem is that nearly all threads are waiting on a single lock here:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;     11 __lll_lock_wait,_L_lock_791,pthread_mutex_lock,__gthread_mutex_lock,lock,lock_guard,mongo::repl::(anonymous,mongo::repl::getNextOpTimes,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,operator()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      1 sched_yield,__wt_yield,__wt_writeunlock,__wt_txn_set_commit_timestamp,__wt_txn_set_timestamp,__session_timestamp_transaction,mongo::WiredTigerRecoveryUnit::setTimestamp,mongo::WiredTigerRecordStore::oplogDiskLocRegister,mongo::repl::(anonymous,mongo::repl::getNextOpTimes,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,operator()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Mongod runs 50% faster in 3.5.12 than it does in 3.5.13 (and later, up to today&apos;s master).  3.5.12 gets about 67k ops/sec and 3.5.13+ gets 45k ops/sec.  Current master also gets 45k ops/sec.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=milkie&quot; class=&quot;user-hover&quot; rel=&quot;milkie&quot;&gt;milkie&lt;/a&gt; the stacks above show where the bottleneck is.  I see threads waiting on the mutex while one thread is settings its first commit timestamp on a transaction.  &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=michael.cahill&quot; class=&quot;user-hover&quot; rel=&quot;michael.cahill&quot;&gt;michael.cahill&lt;/a&gt; has a diff that pulls the starting of a transaction outside the mutex in &lt;tt&gt;getNextOpTimes&lt;/tt&gt;.  That has a large positive effect on performance.  I have attached it  &lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/attachment/170468/170468_ensure-session.patch&quot; title=&quot;ensure-session.patch attached to SERVER-31694&quot;&gt;ensure-session.patch&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.mongodb.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt; . It needs review, naming and cleanup but shows the idea.  &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=michael.cahill&quot; class=&quot;user-hover&quot; rel=&quot;michael.cahill&quot;&gt;michael.cahill&lt;/a&gt;&apos;s numbers show this patch gets almost 50% speedup (master 64k ops/sec, patched 93k ops/sec).&lt;/p&gt;

</comment>
                            <comment id="1718179" author="michael.cahill" created="Mon, 6 Nov 2017 23:01:36 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sue.loverso&quot; class=&quot;user-hover&quot; rel=&quot;sue.loverso&quot;&gt;sue.loverso&lt;/a&gt;, thanks for the update.  The changes you are suggesting to &lt;tt&gt;__wt_txn_global_set_timestamp&lt;/tt&gt; are in the &lt;a href=&quot;https://jira.mongodb.org/browse/WT-3715&quot; title=&quot;Performance tuning for cache overflow mechanism&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-3715&quot;&gt;&lt;del&gt;WT-3715&lt;/del&gt;&lt;/a&gt; branch.&lt;/p&gt;</comment>
                            <comment id="1718062" author="sue.loverso" created="Mon, 6 Nov 2017 21:49:34 +0000"  >&lt;p&gt;Last week I ran this test against successive releases from 3.5.9-3.5.13.  Each showed a gradual decline but then there was a large drop between 3.5.12 and 3.5.13.  There has been more, gradual code added that uses timestamps in that period as well.&lt;/p&gt;
&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;th class=&apos;confluenceTh&apos;&gt; Release &lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt; insert ops/sec &lt;/th&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 3.5.9 &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 80k &lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 3.5.11 &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 76k &lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt;3.5.12 &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 66k &lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 3.5.13 &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 46k &lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;


&lt;p&gt;I instrumented a whole bunch of temporary statistics (to 3.5.13) to characterize this today.  Here are some results.  It builds on the write locks mentioned above by Bruce but with a lot more detail.&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;Every transaction takes the &lt;tt&gt;txn_global-&amp;gt;commit_timestamp_rwlock&lt;/tt&gt; for writing twice (once to insert into the queue, once to remove).&lt;/li&gt;
	&lt;li&gt;Acquiring that rwlock for writing fails 1.3% of the time (consistently).&lt;/li&gt;
	&lt;li&gt;Nearly every commit takes the &lt;tt&gt;txn_global-&amp;gt;rwlock&lt;/tt&gt; for writing.&lt;/li&gt;
	&lt;li&gt;Acquiring the global rwlock in commit is 66% of the total acquisitions.&lt;/li&gt;
	&lt;li&gt;Every call to &lt;tt&gt;conn-&amp;gt;set_timestamp&lt;/tt&gt; takes the &lt;tt&gt;txn_global-&amp;gt;rwlock&lt;/tt&gt; for writing.&lt;/li&gt;
	&lt;li&gt;Acquiring the global rwlock for writing in &lt;tt&gt;wt_txn_global_set_timestamp&lt;/tt&gt; fails 2% of the time (consistently).&lt;/li&gt;
	&lt;li&gt;Only 1.7% of calls to &lt;tt&gt;wt_txn_global_set_timestamp&lt;/tt&gt; result in actually updating any of the global timestamps.&lt;/li&gt;
	&lt;li&gt;Acquiring the global rwlock in &lt;tt&gt;wt_txn_global_set_timestamp&lt;/tt&gt; is the remaining 34% of the total lock acquisitions.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;The lowest hanging fruit here seems to be in &lt;tt&gt;wt_txn_global_set_timestamp&lt;/tt&gt;.  That is called from replication code and it always, unconditionally acquires the &lt;tt&gt;txn_global-&amp;gt;rwlock&lt;/tt&gt; for writing and it does not need to update 98.3% of the time.  If we could avoid taking that lock until we&apos;re actually needing to update, that should help particularly if the replication stack is holding a lock in its own code while setting the timestamp.&lt;/p&gt;</comment>
                            <comment id="1716182" author="sue.loverso" created="Fri, 3 Nov 2017 02:49:22 +0000"  >&lt;p&gt;Thanks for the explanations!  While &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-30845&quot; title=&quot;Avoid updating the stable timestamp in WiredTiger unnecessarily&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-30845&quot;&gt;&lt;del&gt;SERVER-30845&lt;/del&gt;&lt;/a&gt; that &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=milkie&quot; class=&quot;user-hover&quot; rel=&quot;milkie&quot;&gt;milkie&lt;/a&gt; mentioned is one path, I&apos;m sure the locks inside WiredTiger are the long pole here.  As I&apos;ve been testing each release I&apos;m seeing gradual degradation in time (with the bigger leap at 3.5.13) so it is likely that the gradual addition of more code using timestamps has increased the contention each 3.5.N release.  I have new ideas for experiments I want to run with this.&lt;/p&gt;</comment>
                            <comment id="1716134" author="william.schultz" created="Thu, 2 Nov 2017 23:36:39 +0000"  >&lt;p&gt;As Andy mentioned, my commit &lt;a href=&quot;https://github.com/mongodb/mongo/commit/c7661b14867cd058e1a67986b8e05a7020fc0a5e&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;c7661b14867cd058e1a67986b8e05a7020fc0a5e &lt;/a&gt; did introduce a considerable regression, which was noted by BF-6399. It was resolved, however, by commit &lt;a href=&quot;https://github.com/mongodb/mongo/commit/bf8ae1b3edbf7470e92b7407f76a042cc2246d48&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;bf8ae1b3edbf7470e92b7407f76a042cc2246d48&lt;/a&gt;. Even with that fix, though, it could still be investigated as a possible source of the regression against 3.4. Perhaps we should determine if the slowest part of the &quot;set stable timestamp&quot; logic lies at the replication layer (i.e. &lt;tt&gt;ReplicationCoordinatorImpl::_setStableTimestampForStorage_inlock&lt;/tt&gt;), or at the storage layer (&lt;tt&gt;StorageInterface::setStableTimestamp&lt;/tt&gt;)?&lt;/p&gt;</comment>
                            <comment id="1716116" author="milkie" created="Thu, 2 Nov 2017 22:58:48 +0000"  >&lt;p&gt;There is also &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-30845&quot; title=&quot;Avoid updating the stable timestamp in WiredTiger unnecessarily&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-30845&quot;&gt;&lt;del&gt;SERVER-30845&lt;/del&gt;&lt;/a&gt;.&lt;/p&gt;</comment>
                            <comment id="1716114" author="schwerin" created="Thu, 2 Nov 2017 22:56:53 +0000"  >&lt;p&gt;In those stacks, there is a thread blocked on a WT spinlock trying to set the &quot;oldest timestamp&quot; / &quot;stable timestamp for storage&quot; while holding the replication coordinator mutex. If that&apos;s actually taking a long time, and not a coincidence, then it is causing other inserts to block waiting to do the same (7 threads) or to assign a term to their oplog entries (2 threads).&lt;/p&gt;

&lt;p&gt;That makes &lt;a href=&quot;https://github.com/mongodb/mongo/commit/c7661b14867cd058e1a67986b8e05a7020fc0a5e&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;c7661b14867cd058e1a67986b8e05a7020fc0a5e&lt;/a&gt; by &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=william.schultz&quot; class=&quot;user-hover&quot; rel=&quot;william.schultz&quot;&gt;william.schultz&lt;/a&gt; a candidate. It committed on August 8, and so the first release it appeared in was 3.5.12.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sue.loverso&quot; class=&quot;user-hover&quot; rel=&quot;sue.loverso&quot;&gt;sue.loverso&lt;/a&gt;, is it hard to run your test at and immediately before this commit? If not, could you?&lt;/p&gt;

&lt;p&gt;If this is the problem, then our goal is probably to speed up _storage-&amp;gt;setStableTimestamp, to shorten the critical section. &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=redbeard0531&quot; class=&quot;user-hover&quot; rel=&quot;redbeard0531&quot;&gt;redbeard0531&lt;/a&gt;, did you look into this earlier when optimizing?&lt;/p&gt;

&lt;p&gt;Does this affect benchmarks other than bulk insert?&lt;/p&gt;</comment>
                            <comment id="1716017" author="sue.loverso" created="Thu, 2 Nov 2017 21:19:44 +0000"  >&lt;p&gt;I have attached the full pmp output files showing all threads that contains the threads above for both 3.5.9 and 3.5.13.&lt;/p&gt;</comment>
                            <comment id="1715975" author="sue.loverso" created="Thu, 2 Nov 2017 20:51:53 +0000"  >&lt;p&gt;Ok, since 3.5.13 the replication initiate stuff changed such that running just &lt;tt&gt;rs.initiate()&lt;/tt&gt; was failing and then running the script resulted in the 14k ops/sec in some hobbled mode.  Correctly starting replication I&apos;m seeing about 46k ops/sec.  Much better but far less than the 80k I see consistently in 3.5.9.  Running pmp.sh on during the test, I see most/all threads in the WiredTiger library inserting items in 3.5.9 and most insert threads waiting for a replication lock in 3.5.13.  If I &lt;tt&gt;grep receivedInsert pmp.out&lt;/tt&gt; as the way to see what the 15 application threads are doing I see in 3.5.9:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;     14 select,__wt_sleep,__wt_page_in_func,__wt_page_swap_func,__wt_row_search,__cursor_row_search,__wt_btcur_insert,__curfile_insert,mongo::WiredTigerRecordStore::_insertRecords,mongo::WiredTigerRecordStore::insertRecords,mongo::CollectionImpl::_insertDocuments,mongo::CollectionImpl::insertDocuments,insertDocuments,mongo::insertDocuments,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::processMessage,mongo::ServiceStateMachine::runNext,operator(),std::_Function_handler&amp;lt;void(),,operator(),mongo::(anonymous,start_thread,clone&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      1 __split_insert,__wt_split_insert,__evict_review,__wt_evict,__wt_page_release_evict,__wt_page_in_func,__wt_page_swap_func,__wt_row_search,__cursor_row_search,__wt_btcur_insert,__curfile_insert,mongo::WiredTigerRecordStore::_insertRecords,mongo::WiredTigerRecordStore::insertRecords,mongo::CollectionImpl::_insertDocuments,mongo::CollectionImpl::insertDocuments,insertDocuments,mongo::insertDocuments,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::processMessage,mongo::ServiceStateMachine::runNext,operator(),std::_Function_handler&amp;lt;void(),,operator(),mongo::(anonymous,start_thread,clone&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;But in 3.5.13, 11 out of the 15 threads are waiting on one or more locks in mongoDB and only a single thread is inside WiredTiger:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      7 __lll_lock_wait,_L_lock_791,pthread_mutex_lock,__gthread_mutex_lock,lock,lock,unique_lock,mongo::repl::ReplicationCoordinatorImpl::setMyLastAppliedOpTimeForward,operator(),mongo::RecoveryUnit::OnCommitChange::commit,mongo::repl::_logOpsInner,mongo::WiredTigerRecoveryUnit::_commit,commit,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,mongo::ServiceStateMachine::runNext,operator(),mongo::transport::ServiceExecutorSynchronous::schedule(std::function&amp;lt;void,_scheduleFunc&amp;lt;mongo::ServiceStateMachine::scheduleNext(mongo::transport::ServiceExecutor::ScheduleFlags)::&amp;lt;lambda()&amp;gt;,scheduleNext,mongo::ServiceStateMachine::_sourceCallback,mongo::ServiceStateMachine::_runNextInGuard,mongo::ServiceStateMachine::runNext,operator(),operator(),std::_Function_handler&amp;lt;void(),,operator(),mongo::(anonymous,start_thread,clone&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      2 __lll_lock_wait,_L_lock_791,pthread_mutex_lock,__gthread_mutex_lock,lock,lock_guard,mongo::repl::ReplicationCoordinatorImpl::getTerm,mongo::repl::(anonymous,mongo::repl::getNextOpTimes,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,mongo::ServiceStateMachine::runNext,operator(),mongo::transport::ServiceExecutorSynchronous::schedule(std::function&amp;lt;void,_scheduleFunc&amp;lt;mongo::ServiceStateMachine::scheduleNext(mongo::transport::ServiceExecutor::ScheduleFlags)::&amp;lt;lambda()&amp;gt;,scheduleNext,mongo::ServiceStateMachine::_sourceCallback,mongo::ServiceStateMachine::_runNextInGuard,mongo::ServiceStateMachine::runNext,operator(),operator(),std::_Function_handler&amp;lt;void(),,operator(),mongo::(anonymous,start_thread,clone&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      1 tcmalloc::ThreadCache::FetchFromCentralCache,Allocate,do_malloc_small,do_malloc,cpp_alloc,tc_new,mongo::WiredTigerRecordStore::OplogStones::updateCurrentStoneAfterInsertOnCommit,mongo::WiredTigerRecordStore::_insertRecords,mongo::WiredTigerRecordStore::insertRecordsWithDocWriter,mongo::CollectionImpl::insertDocumentsForOplog,insertDocumentsForOplog,mongo::repl::_logOpsInner,mongo::repl::logInsertOps,mongo::OpObserverImpl::onInserts,mongo::CollectionImpl::insertDocuments,insertDocuments,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,mongo::ServiceStateMachine::runNext,operator(),mongo::transport::ServiceExecutorSynchronous::schedule(std::function&amp;lt;void,_scheduleFunc&amp;lt;mongo::ServiceStateMachine::scheduleNext(mongo::transport::ServiceExecutor::ScheduleFlags)::&amp;lt;lambda()&amp;gt;,scheduleNext,mongo::ServiceStateMachine::_sourceCallback,mongo::ServiceStateMachine::_runNextInGuard,mongo::ServiceStateMachine::runNext,operator(),operator(),std::_Function_handler&amp;lt;void(),,operator(),mongo::(anonymous,start_thread,clone&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      1 sched_yield,__wt_yield,__wt_readlock,__wt_txn_global_set_timestamp,__conn_set_timestamp,mongo::WiredTigerKVEngine::_setOldestTimestamp,mongo::repl::ReplicationCoordinatorImpl::_setStableTimestampForStorage_inlock,mongo::repl::ReplicationCoordinatorImpl::_setMyLastAppliedOpTime_inlock,mongo::repl::ReplicationCoordinatorImpl::setMyLastAppliedOpTimeForward,operator(),mongo::RecoveryUnit::OnCommitChange::commit,mongo::repl::_logOpsInner,mongo::WiredTigerRecoveryUnit::_commit,commit,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,mongo::ServiceStateMachine::runNext,operator(),mongo::transport::ServiceExecutorSynchronous::schedule(std::function&amp;lt;void,_scheduleFunc&amp;lt;mongo::ServiceStateMachine::scheduleNext(mongo::transport::ServiceExecutor::ScheduleFlags)::&amp;lt;lambda()&amp;gt;,scheduleNext,mongo::ServiceStateMachine::_sourceCallback,mongo::ServiceStateMachine::_runNextInGuard,mongo::ServiceStateMachine::runNext,operator(),operator(),std::_Function_handler&amp;lt;void(),,operator(),mongo::(anonymous,start_thread,clone&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      1 operator==,mongo::WiredTigerIndex::setKey,mongo::WiredTigerIndexUnique::_insert,mongo::WiredTigerIndex::insert,mongo::IndexAccessMethod::insert,mongo::IndexCatalogImpl::_indexFilteredRecords,mongo::IndexCatalogImpl::_indexRecords,mongo::IndexCatalogImpl::indexRecords,indexRecords,mongo::CollectionImpl::_insertDocuments,mongo::CollectionImpl::insertDocuments,insertDocuments,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,mongo::ServiceStateMachine::runNext,operator(),mongo::transport::ServiceExecutorSynchronous::schedule(std::function&amp;lt;void,_scheduleFunc&amp;lt;mongo::ServiceStateMachine::scheduleNext(mongo::transport::ServiceExecutor::ScheduleFlags)::&amp;lt;lambda()&amp;gt;,scheduleNext,mongo::ServiceStateMachine::_sourceCallback,mongo::ServiceStateMachine::_runNextInGuard,mongo::ServiceStateMachine::runNext,operator(),operator(),std::_Function_handler&amp;lt;void(),,operator(),mongo::(anonymous,start_thread,clone&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      1 lock,lock_guard,mongo::LockerImpl&amp;lt;false&amp;gt;::getLockMode,mongo::LockerImpl&amp;lt;false&amp;gt;::isLocked,mongo::repl::ReplicationCoordinatorImpl::canAcceptWritesFor,mongo::repl::ReplicationCoordinatorImpl::shouldRelaxIndexConstraints,mongo::IndexCatalogImpl::prepareInsertDeleteOptions,mongo::IndexCatalogImpl::_indexFilteredRecords,mongo::IndexCatalogImpl::_indexRecords,mongo::IndexCatalogImpl::indexRecords,indexRecords,mongo::CollectionImpl::_insertDocuments,mongo::CollectionImpl::insertDocuments,insertDocuments,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,mongo::ServiceStateMachine::runNext,operator(),mongo::transport::ServiceExecutorSynchronous::schedule(std::function&amp;lt;void,_scheduleFunc&amp;lt;mongo::ServiceStateMachine::scheduleNext(mongo::transport::ServiceExecutor::ScheduleFlags)::&amp;lt;lambda()&amp;gt;,scheduleNext,mongo::ServiceStateMachine::_sourceCallback,mongo::ServiceStateMachine::_runNextInGuard,mongo::ServiceStateMachine::runNext,operator(),operator(),std::_Function_handler&amp;lt;void(),,operator(),mongo::(anonymous,start_thread,clone&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      1 __lll_lock_wait,_L_lock_791,pthread_mutex_lock,__gthread_mutex_lock,lock,lock_guard,mongo::repl::(anonymous,mongo::repl::getNextOpTimes,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,mongo::ServiceStateMachine::runNext,operator(),mongo::transport::ServiceExecutorSynchronous::schedule(std::function&amp;lt;void,_scheduleFunc&amp;lt;mongo::ServiceStateMachine::scheduleNext(mongo::transport::ServiceExecutor::ScheduleFlags)::&amp;lt;lambda()&amp;gt;,scheduleNext,mongo::ServiceStateMachine::_sourceCallback,mongo::ServiceStateMachine::_runNextInGuard,mongo::ServiceStateMachine::runNext,operator(),operator(),std::_Function_handler&amp;lt;void(),,operator(),mongo::(anonymous,start_thread,clone&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;      1 GetOrDefault,GetSizeClassIfCached,do_free_helper,do_free_with_callback,do_free,tc_free,~intrusive_ptr,~SharedBuffer,~ConstSharedBuffer,~BSONObj,~OplogDocWriter,_Destroy&amp;lt;mongo::repl::(anonymous,__destroy&amp;lt;mongo::repl::(anonymous,_Destroy&amp;lt;mongo::repl::(anonymous,_Destroy&amp;lt;mongo::repl::(anonymous,~vector,mongo::repl::logInsertOps,mongo::OpObserverImpl::onInserts,mongo::CollectionImpl::insertDocuments,insertDocuments,mongo::(anonymous,operator(),writeConflictRetry&amp;lt;mongo::(anonymous,insertBatchAndHandleErrors,mongo::performInserts,receivedInsert,mongo::ServiceEntryPointMongod::handleRequest,mongo::ServiceStateMachine::_processMessage,mongo::ServiceStateMachine::_runNextInGuard,mongo::ServiceStateMachine::runNext,operator(),mongo::transport::ServiceExecutorSynchronous::schedule(std::function&amp;lt;void,_scheduleFunc&amp;lt;mongo::ServiceStateMachine::scheduleNext(mongo::transport::ServiceExecutor::ScheduleFlags)::&amp;lt;lambda()&amp;gt;,scheduleNext,mongo::ServiceStateMachine::_sourceCallback,mongo::ServiceStateMachine::_runNextInGuard,mongo::ServiceStateMachine::runNext,operator(),operator(),std::_Function_handler&amp;lt;void(),,operator(),mongo::(anonymous,start_thread,clone&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=spencer&quot; class=&quot;user-hover&quot; rel=&quot;spencer&quot;&gt;spencer&lt;/a&gt; can you take a look and re-assign if needed?&lt;/p&gt;</comment>
                            <comment id="1715562" author="sue.loverso" created="Thu, 2 Nov 2017 16:27:16 +0000"  >&lt;blockquote&gt;&lt;p&gt;could you please retest against the &lt;a href=&quot;https://jira.mongodb.org/browse/WT-3715&quot; title=&quot;Performance tuning for cache overflow mechanism&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-3715&quot;&gt;&lt;del&gt;WT-3715&lt;/del&gt;&lt;/a&gt; branch to see whether it improves things? &lt;/p&gt;&lt;/blockquote&gt;

&lt;p&gt;Running on that branch with MongoDB master yields the same throughput, about 14100 ops/sec.  So there is no change compared to the earlier run I did last week.  Continuing to try to isolate when in the timeline the large drop happened and then get stacks and isolate further.&lt;/p&gt;</comment>
                            <comment id="1715516" author="sue.loverso" created="Thu, 2 Nov 2017 15:57:48 +0000"  >&lt;p&gt;Yes, I&apos;ve been running tests today trying to isolate when it started dropping so much, but I&apos;ll retest with the branch first.&lt;/p&gt;</comment>
                            <comment id="1715463" author="michael.cahill" created="Thu, 2 Nov 2017 15:29:50 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=sue.loverso&quot; class=&quot;user-hover&quot; rel=&quot;sue.loverso&quot;&gt;sue.loverso&lt;/a&gt;, could you please retest against the &lt;a href=&quot;https://jira.mongodb.org/browse/WT-3715&quot; title=&quot;Performance tuning for cache overflow mechanism&quot; class=&quot;issue-link&quot; data-issue-key=&quot;WT-3715&quot;&gt;&lt;del&gt;WT-3715&lt;/del&gt;&lt;/a&gt; branch to see whether it improves things?  There are some changes in that branch that avoid acquiring the timestamp rwlock when we&apos;re not going to update anything.&lt;/p&gt;

&lt;p&gt;Also, the lookaside eviction changes probably explain most of the additional regression when replication is enabled, so that should also be better on the branch.&lt;/p&gt;

&lt;p&gt;Thanks!&lt;/p&gt;</comment>
                            <comment id="1710722" author="sue.loverso" created="Thu, 26 Oct 2017 23:41:51 +0000"  >&lt;p&gt;Here are the results I ran today.  I ran 3.5.8 with the current WiredTiger develop branch but it showed the same numbers as straight 3.5.8.  I will further investigate the timestamp and replication issue above and given my much worse numbers, I&apos;m sure it&apos;ll be easy to get stacks.  I will run with successive releases because the timestamp work was rolled out gradually.  The numbers below are the average of 3 runs, but they were not statistically far apart run-to-run.&lt;/p&gt;

&lt;div class=&apos;table-wrap&apos;&gt;
&lt;table class=&apos;confluenceTable&apos;&gt;&lt;tbody&gt;
&lt;tr&gt;
&lt;th class=&apos;confluenceTh&apos;&gt; Mode &lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt; 3.4.9 &lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt; 3.5.8 &lt;/th&gt;
&lt;th class=&apos;confluenceTh&apos;&gt; MongoDB Master + WT develop &lt;/th&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; Standalone &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 205497 &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 130524 (-37%) &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 115737 (-44%) &lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 1 node repl &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 126561 &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 88990 (-30%) &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; 14131 (-89%) &lt;/td&gt;
&lt;/tr&gt;
&lt;tr&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; Standalone/repl diff &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; -39% &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; -32% &lt;/td&gt;
&lt;td class=&apos;confluenceTd&apos;&gt; -88% &lt;/td&gt;
&lt;/tr&gt;
&lt;/tbody&gt;&lt;/table&gt;
&lt;/div&gt;

</comment>
                            <comment id="1710684" author="sue.loverso" created="Thu, 26 Oct 2017 22:17:09 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt; thank you for those stacks.  I ran eight variations today and I&apos;ll post the full results soon.  But your stacks give a strong clue for where we should start.  FYI I ran MongoDB-3.5.8+current WiredTiger-develop and MongoDB-master+WiredTiger-develop.  Master is running 84% slower than 3.5.8 (14100 ops/sec versus 88000 ops/sec).  &lt;/p&gt;</comment>
                            <comment id="1710457" author="bruce.lucas@10gen.com" created="Thu, 26 Oct 2017 19:06:28 +0000"  >&lt;p&gt;I looked at the ftdc data from the test and confirmed that in 3.6.0-rc0 there are a large number of write-lock calls for the replica set case, but not for the standalone case, confirming that it is a possible suspect for the regression seen in the former but not the latter.&lt;/p&gt;

&lt;p&gt;Using gdb determined that we&apos;re calling __wt_writelock in the following 5 places in equal numbers, apparently each once per update:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__wt_writelock&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__wt_txn_clear_commit_timestamp&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__wt_txn_release&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__wt_txn_commit&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__session_commit_transaction&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::WiredTigerRecoveryUnit::_txnClose&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::WiredTigerRecoveryUnit::_commit&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::UpdateStage::transformAndUpdate&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__wt_writelock&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__wt_txn_commit&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__session_commit_transaction&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::WiredTigerRecoveryUnit::_txnClose&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::WiredTigerRecoveryUnit::_commit&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::UpdateStage::transformAndUpdate&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__wt_writelock&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__wt_txn_global_set_timestamp&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__conn_set_timestamp&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::WiredTigerKVEngine::_setOldestTimestamp&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::repl::ReplicationCoordinatorImpl::_setStableTimestampForStorage_inlock&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::repl::ReplicationCoordinatorImpl::_setMyLastAppliedOpTime_inlock&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::repl::ReplicationCoordinatorImpl::setMyLastAppliedOpTimeForward&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;void mongo::RecoveryUnit::onCommit&amp;lt;...&amp;gt;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::WiredTigerRecoveryUnit::_commit&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::UpdateStage::transformAndUpdate&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__wt_writelock&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__wt_txn_set_commit_timestamp&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__wt_txn_set_timestamp&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__session_timestamp_transaction&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::WiredTigerRecoveryUnit::setTimestamp&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::WiredTigerRecordStore::oplogDiskLocRegister&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::repl::(anonymous namespace)::_getNextOpTimes&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::repl::logOp&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::OpObserverImpl::onUpdate&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::CollectionImpl::updateDocumentWithDamages&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::UpdateStage::transformAndUpdate&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__wt_writelock&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__wt_txn_update_pinned_timestamp&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__wt_txn_global_set_timestamp&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;__conn_set_timestamp&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::WiredTigerKVEngine::_setOldestTimestamp&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::repl::ReplicationCoordinatorImpl::_setStableTimestampForStorage_inlock&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::repl::ReplicationCoordinatorImpl::_setMyLastAppliedOpTime_inlock&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::repl::ReplicationCoordinatorImpl::setMyLastAppliedOpTimeForward&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;void mongo::RecoveryUnit::onCommit&amp;lt;...&amp;gt;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::WiredTigerRecoveryUnit::_commit&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongo::UpdateStage::transformAndUpdate&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="1707439" author="bruce.lucas@10gen.com" created="Tue, 24 Oct 2017 16:04:41 +0000"  >&lt;p&gt;12 cores / 24 cpus&lt;br/&gt;
SSD disk (but probably not relevant since there are no checkpoints in the run and not specifying j:true)&lt;br/&gt;
cache far from full&lt;/p&gt;</comment>
                            <comment id="1707436" author="dan@10gen.com" created="Tue, 24 Oct 2017 16:01:52 +0000"  >&lt;p&gt;what&apos;s your machine setup?&lt;/p&gt;</comment>
                            <comment id="1707360" author="bruce.lucas@10gen.com" created="Tue, 24 Oct 2017 15:35:37 +0000"  >&lt;p&gt;It can use either depending on value of writeCmd option; ops/s is different depending on which you choose but regression is essentially the same.&lt;/p&gt;</comment>
                            <comment id="1707348" author="milkie" created="Tue, 24 Oct 2017 15:29:13 +0000"  >&lt;p&gt;How does this workload compare to the insert Microbenchmark tests and the YCSB load sys-perf tests?&lt;br/&gt;
Also, does benchrun use old-style OP_QUERY inserts, or does it use write commands?&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                            <outwardlinks description="depends on">
                                        <issuelink>
            <issuekey id="457004">SERVER-31902</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="456409">SERVER-31879</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="456448">SERVER-31882</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="456888">SERVER-31899</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="457144">SERVER-31912</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="532172">SERVER-34604</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="720222">SERVER-40250</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="1867214">SERVER-59776</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="530866">SERVER-34562</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                                        </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="170468" name="ensure-session.patch" size="1788" author="sue.loverso@mongodb.com" created="Wed, 8 Nov 2017 01:53:23 +0000"/>
                            <attachment id="170078" name="pmp.out.3513" size="17455" author="sue.loverso@mongodb.com" created="Thu, 2 Nov 2017 21:19:11 +0000"/>
                            <attachment id="170079" name="pmp.out.359" size="8482" author="sue.loverso@mongodb.com" created="Thu, 2 Nov 2017 21:19:11 +0000"/>
                            <attachment id="170505" name="pmp.patch.sesscache" size="11982" author="sue.loverso@mongodb.com" created="Wed, 8 Nov 2017 16:26:01 +0000"/>
                            <attachment id="169119" name="run2.png" size="130501" author="bruce.lucas@mongodb.com" created="Tue, 24 Oct 2017 15:04:32 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>52.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>6.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_12751" key="com.atlassian.jira.plugin.system.customfieldtypes:multiselect">
                        <customfieldname>Assigned Teams</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="25128"><![CDATA[Replication]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_13552" key="com.go2group.jira.plugin.crm:crm_generic_field">
                        <customfieldname>Case</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[[5002K00000jcGvqQAE, 5002K00000msFwwQAE]]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 24 Oct 2017 15:29:13 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        1 year, 14 weeks, 5 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[<s><a href='https://jira.mongodb.org/browse/SERVER-31882'>SERVER-31882</a></s>, <s><a href='https://jira.mongodb.org/browse/SERVER-31879'>SERVER-31879</a></s>, <s><a href='https://jira.mongodb.org/browse/SERVER-31899'>SERVER-31899</a></s>, <s><a href='https://jira.mongodb.org/browse/SERVER-31902'>SERVER-31902</a></s>, <s><a href='https://jira.mongodb.org/browse/SERVER-31912'>SERVER-31912</a></s>]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_14262" key="com.atlassian.jira.plugin.system.customfieldtypes:datepicker">
                        <customfieldname>End date</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 1 Nov 2017 23:59:59 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10857" key="com.pyxis.greenhopper.jira:gh-epic-link">
                        <customfieldname>Epic Link</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>PM-1170</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>alexander.golin@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            1 year, 14 weeks, 5 days ago
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_16465" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Linked BF Score</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>stutiredboy@gmail.com</customfieldvalue>
            <customfieldvalue>schwerin@mongodb.com</customfieldvalue>
            <customfieldvalue>backlog-server-repl</customfieldvalue>
            <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>dan@mongodb.com</customfieldvalue>
            <customfieldvalue>bartle</customfieldvalue>
            <customfieldvalue>milkie@mongodb.com</customfieldvalue>
            <customfieldvalue>geert.bosch@mongodb.com</customfieldvalue>
            <customfieldvalue>judah.schvimer@mongodb.com</customfieldvalue>
            <customfieldvalue>michael.cahill@mongodb.com</customfieldvalue>
            <customfieldvalue>sue.loverso@mongodb.com</customfieldvalue>
            <customfieldvalue>victorgp</customfieldvalue>
            <customfieldvalue>william.schultz@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|htikt3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|ht9xf3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="1951">Storage 2017-11-13</customfieldvalue>
    <customfieldvalue id="1978">Storage 2017-12-04</customfieldvalue>
    <customfieldvalue id="2094">Storage 2018-01-29</customfieldvalue>
    <customfieldvalue id="2146">Repl 2018-02-26</customfieldvalue>
    <customfieldvalue id="2185">Repl 2018-03-12</customfieldvalue>
    <customfieldvalue id="2186">Repl 2018-03-26</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_14261" key="com.atlassian.jira.plugin.system.customfieldtypes:datepicker">
                        <customfieldname>Start date</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 24 Oct 2017 00:00:00 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hti6wv:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>