<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:49:12 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-18908] Secondaries unable to keep up with primary under WiredTiger</title>
                <link>https://jira.mongodb.org/browse/SERVER-18908</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;ul&gt;
	&lt;li&gt;hardware: 24 CPUs, 64 GB memory, SSD (all mongods and clients on same machine)&lt;/li&gt;
	&lt;li&gt;start 3-member replica set with following options:
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongod --oplogSize 50 --storageEngine wiredTiger --nojournal --replSet ...&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;Note: repros with journal also, ran without journal to rule out that as a cause.&lt;/p&gt;&lt;/li&gt;
	&lt;li&gt;Simple small document insert workload: 5-16 threads (number doesn&apos;t matter to the repro) each inserting small documents {_id:..., x:0} in batches of 10k&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Replica lag grows unbounded as secondaries process ops at maybe 50-80% the rate of the primary.&lt;/p&gt;

&lt;p&gt;Some stats of note:&lt;/p&gt;

&lt;p&gt;primary&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/77447/77447_pri.png&quot; width=&quot;900&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;secondary&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/77448/77448_sec.png&quot; width=&quot;900&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;op rate on secondary is maybe half that on primary&lt;/li&gt;
	&lt;li&gt;ops in flight (i.e. active queues) is much less even on the secondary, although that isn&apos;t reflected in the reported op rates&lt;/li&gt;
	&lt;li&gt;secondary is executing far more search near calls, about one per document, vs what appears to be about one every 100 documents on primary&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;Will get stack traces.&lt;/p&gt;



</description>
                <environment></environment>
        <key id="209769">SERVER-18908</key>
            <summary>Secondaries unable to keep up with primary under WiredTiger</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="mathias@mongodb.com">Mathias Stearn</assignee>
                                    <reporter username="bruce.lucas@mongodb.com">Bruce Lucas</reporter>
                        <labels>
                            <label>WTcap</label>
                    </labels>
                <created>Wed, 10 Jun 2015 16:21:33 +0000</created>
                <updated>Mon, 31 Oct 2016 13:31:26 +0000</updated>
                            <resolved>Fri, 28 Oct 2016 21:21:44 +0000</resolved>
                                    <version>3.0.3</version>
                    <version>3.0.4</version>
                                    <fixVersion>3.4.0-rc2</fixVersion>
                                    <component>Performance</component>
                    <component>WiredTiger</component>
                                        <votes>11</votes>
                                    <watches>54</watches>
                                                                                                                <comments>
                            <comment id="1421087" author="redbeard0531" created="Fri, 28 Oct 2016 21:21:44 +0000"  >&lt;p&gt;Significant work on secondary performance was done under &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-24242&quot; title=&quot;Improve replication performance&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-24242&quot;&gt;&lt;del&gt;SERVER-24242&lt;/del&gt;&lt;/a&gt;. That concludes the work we intend to do for 3.4. If this continues to be an issue, please open a new ticket.&lt;/p&gt;</comment>
                            <comment id="1123235" author="jzou@ebay.com" created="Tue, 29 Dec 2015 07:06:38 +0000"  >&lt;p&gt;It would be very appreciated if you can share how 3.2.0 was improved in pipelining and parallelism(some implementation details). To be honest, it would be very good show cases or new features for marketing.&lt;/p&gt;

&lt;p&gt;Beside, serious applications/shops will always 1-2 major release behind the bleeding-edge release,  backporting these improvements to mongo 3.0.x is highly demanded in many shops. &lt;/p&gt;
</comment>
                            <comment id="1115312" author="martin.bligh" created="Tue, 15 Dec 2015 20:33:20 +0000"  >&lt;p&gt;The code has been changed significantly since that comment was written.&lt;br/&gt;
Though there is still other work we will look at, the implementation in 3.2.0 is much improved, with much more pipelining and parallelism.&lt;/p&gt;</comment>
                            <comment id="1115293" author="jzou@ebay.com" created="Tue, 15 Dec 2015 20:26:18 +0000"  >&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;On the primary, each connection thread writes its own ops to the oplog, which proceed in parallel under WT with document-level concurrency. On the secondary, writing to the oplog is done sequentially by the sync thread, making it a serial bottleneck that doesn&apos;t exist on the primary. Under mmapv1 with collection-level locking, writing to the oplog is a serial bottlenck on both the primary and the secondary.&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;In the following 24 samples, runSyncThread spent 67% of its time in applyOps waiting for the n worker threads to finish applying the ops, and 29% of its time writing those ops to the oplog sequentially. This is about the right ratio to account for the performance difference we see between primary and secondary.&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;This is very inefficient implementation of replication. how about split up the replication task to multiple threads with minimal sequential operations: ex:&lt;br/&gt;
1. runSyncThread is dedicated thread(or pool of threads) that only takes care of oplog shipping, don&apos;t need to wait the log apply threads, what they have to do is simply copy the oplog from source(you can add compression function if needed to save bandwidth).&lt;br/&gt;
2  apply threads, oplog apply main thread + multiple apply worker threads. apply main thread just scan through oplog and distribute entries to concurrent apply worker threads that split up their jobs by collection + _id.&lt;/p&gt;</comment>
                            <comment id="1028808" author="xgen-internal-githook" created="Mon, 14 Sep 2015 14:16:52 +0000"  >&lt;p&gt;Author:&lt;/p&gt;
{u&apos;username&apos;: u&apos;martinbligh&apos;, u&apos;name&apos;: u&apos;Martin Bligh&apos;, u&apos;email&apos;: u&apos;mbligh@mongodb.com&apos;}
&lt;p&gt;Message: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-18908&quot; title=&quot;Secondaries unable to keep up with primary under WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-18908&quot;&gt;&lt;del&gt;SERVER-18908&lt;/del&gt;&lt;/a&gt;: Write secondary oplog as a vector&lt;br/&gt;
Branch: master&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/21c6a574d2cd497089d41d1b418fdfdabf7fc953&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/21c6a574d2cd497089d41d1b418fdfdabf7fc953&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="940981" author="bruce.lucas@10gen.com" created="Mon, 15 Jun 2015 19:56:28 +0000"  >&lt;p&gt;I&apos;ve opened &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-18982&quot; title=&quot;Apply replicated inserts as inserts&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-18982&quot;&gt;&lt;del&gt;SERVER-18982&lt;/del&gt;&lt;/a&gt; to track processing inserts as inserts, and &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-18983&quot; title=&quot;Process oplog inserts, and applying, on the secondary in parallel&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-18983&quot;&gt;&lt;del&gt;SERVER-18983&lt;/del&gt;&lt;/a&gt; to track parallel processing of oplog inserts.&lt;/p&gt;</comment>
                            <comment id="940817" author="bruce.lucas@10gen.com" created="Mon, 15 Jun 2015 18:28:51 +0000"  >&lt;p&gt;The number of replication worker threads on the secondary is hard-coded at 16. This can become a bottleneck creating replication lag if the primary has more concurrency than that. For example, with 12 application threads, the following replication lags were measured at the end of a 110-second run for varying numbers of replication worker threads:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;worker	repl&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;threads	lag&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;8	30&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;16	17&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;24	11&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;To address this issue we can implement &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-15344&quot; title=&quot;Make replWriter thread pool size tunable&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-15344&quot;&gt;&lt;del&gt;SERVER-15344&lt;/del&gt;&lt;/a&gt; to make the number of worker threads tunable.&lt;/p&gt;</comment>
                            <comment id="940680" author="bruce.lucas@10gen.com" created="Mon, 15 Jun 2015 17:01:22 +0000"  >&lt;p&gt;Did some more careful measurements evaluating two changes:&lt;/p&gt;

&lt;ol&gt;
	&lt;li&gt;process collection inserts from primary as inserts on secondary, instead of upserts as is currently done&lt;/li&gt;
	&lt;li&gt;disable insertion into oplog on secondary to approximate speedup expected by doing oplog insertion in parallel on worker threads instead of sequentially on sync thread&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;In an insert workload taking about 90 seconds:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;baseline, with neither patch, replication lag grew steadily to 36 seconds&lt;/li&gt;
	&lt;li&gt;with first patch processing collection inserts as inserts: replication lag grew steadily to 21 seconds&lt;/li&gt;
	&lt;li&gt;with second patch disabling insertion into oplog: replication lag grew steadily to 22 seconds&lt;/li&gt;
	&lt;li&gt;with both patches: replication hovered around 0-4 seconds and did not appear to grow.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;So it appears that a combination of both patches, if fully implemented, would address this issue.&lt;/p&gt;

&lt;p&gt;(Note that these results were obtained with 8 application threads. Since the number of replication worker threads is hard-coded at 16, in this situation the time spent inserting documents into the collection may actually be less on the secondary than it was on the primary. This means that although the two patches described eliminated growing replication lag in this case, the better performance on the secondary for the inserts than on the primary may be hiding other inefficiencies on the secondary.)&lt;/p&gt;</comment>
                            <comment id="939105" author="bruce.lucas@10gen.com" created="Fri, 12 Jun 2015 17:02:03 +0000"  >&lt;p&gt;On the primary, each connection thread writes its own ops to the oplog, which proceed in parallel under WT with document-level concurrency. On the secondary, writing to the oplog is done sequentially by the sync thread, making it a serial bottleneck that doesn&apos;t exist on the primary. Under mmapv1 with collection-level locking, writing to the oplog is a serial bottlenck on both the primary and the secondary.&lt;/p&gt;

&lt;p&gt;In the following 24 samples, runSyncThread spent 67% of its time in applyOps waiting for the n worker threads to finish applying the ops, and 29% of its time writing those ops to the oplog sequentially. This is about the right ratio to account for the performance difference we see between primary and secondary.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/77772/77772_log.png&quot; width=&quot;600&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;</comment>
                            <comment id="936717" author="milkie" created="Wed, 10 Jun 2015 18:12:17 +0000"  >&lt;p&gt;One idea we had is that instead of doing upserts for inserts, we could simply do real inserts and ignore duplicate key errors.  The idempotency logic should still work.&lt;/p&gt;</comment>
                            <comment id="936662" author="bruce.lucas@10gen.com" created="Wed, 10 Jun 2015 17:56:02 +0000"  >&lt;p&gt;adapted test to mmpav1: 8 threads inserting into 8 collections&lt;br/&gt;
rough numbers on this test from eyeballing mongostat:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;       pri    sec&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;wt     150k/s 100k/s&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mmapv1  60k/s  60k/s&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;The mmapv1 secondary did appear slightly slower, 5% at most from eyeballing mongostat&lt;br/&gt;
and replication lag did grow, albeit &lt;em&gt;much&lt;/em&gt; more slowly that with WT&lt;/p&gt;</comment>
                            <comment id="936644" author="bruce.lucas@10gen.com" created="Wed, 10 Jun 2015 17:45:28 +0000"  >&lt;p&gt;The search near appers to be because on the secondary we do inserts as upserts to make oplog replay idempotent, and that requires a lookup of the _id. If this is the cause (TBD whether it is) could be related to &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-18823&quot; title=&quot;WT find performance is 1/2 that of MMv1 when using a multikey index&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-18823&quot;&gt;&lt;del&gt;SERVER-18823&lt;/del&gt;&lt;/a&gt;, where we also see __wt_btcur_search_near playing a role.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/77467/77467_search_near.png&quot; width=&quot;700&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;</comment>
                            <comment id="936567" author="milkie" created="Wed, 10 Jun 2015 16:56:57 +0000"  >&lt;p&gt;What does the lag look like for the same parameters except using mmapv1?&lt;br/&gt;
I wonder if this is simply a replication batching algorithm problem.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                            <outwardlinks description="depends on">
                                        <issuelink>
            <issuekey id="212870">SERVER-19122</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="214629">WT-1989</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="210754">SERVER-18983</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="159521">SERVER-15344</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="210752">SERVER-18982</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                            <outwardlinks description="duplicates">
                                                        </outwardlinks>
                                                                <inwardlinks description="is duplicated by">
                                        <issuelink>
            <issuekey id="239227">SERVER-21444</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="223453">SERVER-19668</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="243424">SERVER-21858</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="79761" name="insert-3.0.4.patch" size="2885" author="bruce.lucas@mongodb.com" created="Sun, 28 Jun 2015 23:46:26 +0000"/>
                            <attachment id="79762" name="insert-3.1.4.patch" size="1840" author="bruce.lucas@mongodb.com" created="Sun, 28 Jun 2015 23:46:26 +0000"/>
                            <attachment id="77783" name="lag-02.html" size="1014476" author="bruce.lucas@mongodb.com" created="Fri, 12 Jun 2015 19:22:34 +0000"/>
                            <attachment id="77784" name="lag-03.html" size="507674" author="bruce.lucas@mongodb.com" created="Fri, 12 Jun 2015 19:22:34 +0000"/>
                            <attachment id="77785" name="lag-04.html" size="787159" author="bruce.lucas@mongodb.com" created="Fri, 12 Jun 2015 19:22:34 +0000"/>
                            <attachment id="79759" name="lagC-3.0.4.patch" size="18411" author="bruce.lucas@mongodb.com" created="Sun, 28 Jun 2015 23:46:26 +0000"/>
                            <attachment id="79760" name="lagC-3.1.4.patch" size="17755" author="bruce.lucas@mongodb.com" created="Sun, 28 Jun 2015 23:46:26 +0000"/>
                            <attachment id="80057" name="lagD-3.0.4.patch" size="20207" author="bruce.lucas@mongodb.com" created="Tue, 30 Jun 2015 17:49:07 +0000"/>
                            <attachment id="77772" name="log.png" size="61532" author="bruce.lucas@mongodb.com" created="Fri, 12 Jun 2015 17:01:38 +0000"/>
                            <attachment id="77447" name="pri.png" size="101966" author="bruce.lucas@mongodb.com" created="Wed, 10 Jun 2015 16:21:33 +0000"/>
                            <attachment id="77467" name="search_near.png" size="242829" author="bruce.lucas@mongodb.com" created="Wed, 10 Jun 2015 17:45:03 +0000"/>
                            <attachment id="77448" name="sec.png" size="106804" author="bruce.lucas@mongodb.com" created="Wed, 10 Jun 2015 16:21:33 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>13.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>7.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 10 Jun 2015 16:56:57 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        7 years, 15 weeks, 5 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[<s><a href='https://jira.mongodb.org/browse/SERVER-15344'>SERVER-15344</a></s>, <s><a href='https://jira.mongodb.org/browse/SERVER-18982'>SERVER-18982</a></s>, <s><a href='https://jira.mongodb.org/browse/SERVER-18983'>SERVER-18983</a></s>, <s><a href='https://jira.mongodb.org/browse/SERVER-19122'>SERVER-19122</a></s>, <s><a href='https://jira.mongodb.org/browse/WT-1989'>WT-1989</a></s>]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>false</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            7 years, 15 weeks, 5 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>milkie@mongodb.com</customfieldvalue>
            <customfieldvalue>xgen-internal-githook</customfieldvalue>
            <customfieldvalue>jzou@ebay.com</customfieldvalue>
            <customfieldvalue>martin.bligh</customfieldvalue>
            <customfieldvalue>mathias@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrkrtr:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrc8jb:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="635">Quint Iteration 7</customfieldvalue>
    <customfieldvalue id="705">QuInt A (10/12/15)</customfieldvalue>
    <customfieldvalue id="765">QuInt C (11/23/15)</customfieldvalue>
    <customfieldvalue id="1022">Repl 2016-08-29</customfieldvalue>
    <customfieldvalue id="1231">Repl 2016-09-19</customfieldvalue>
    <customfieldvalue id="1232">Repl 2016-10-10</customfieldvalue>
    <customfieldvalue id="1233">Repl 2016-10-31</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrzwav:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>