<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:58:38 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-21858] A high throughput update workload in a replica set can cause starvation of secondary reads</title>
                <link>https://jira.mongodb.org/browse/SERVER-21858</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Under an update only workload on the primary it is possible to starve out readers on a secondary and cause large replication delays.&lt;/p&gt;

&lt;p&gt;Workload:&lt;br/&gt;
On Primary:&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;1000 small updates (integer sets, increments and an unset)&lt;/li&gt;
	&lt;li&gt;10 massive (500kb) updates&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;On Secondary:&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;10000 findOne&apos;s&lt;/li&gt;
&lt;/ol&gt;

</description>
                <environment></environment>
        <key id="243424">SERVER-21858</key>
            <summary>A high throughput update workload in a replica set can cause starvation of secondary reads</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="3">Duplicate</resolution>
                                        <assignee username="backlog-server-execution">Backlog - Storage Execution Team</assignee>
                                    <reporter username="david.hows">David Hows</reporter>
                        <labels>
                    </labels>
                <created>Fri, 11 Dec 2015 05:07:07 +0000</created>
                <updated>Tue, 6 Dec 2022 04:37:48 +0000</updated>
                            <resolved>Thu, 21 Dec 2017 21:29:56 +0000</resolved>
                                    <version>3.0.7</version>
                    <version>3.2.0</version>
                                                    <component>Replication</component>
                    <component>WiredTiger</component>
                                        <votes>3</votes>
                                    <watches>20</watches>
                                                                                                                <comments>
                            <comment id="1758241" author="asya" created="Thu, 21 Dec 2017 21:29:56 +0000"  >&lt;p&gt;Duplicate of &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-20328&quot; title=&quot;Allow secondary reads while applying oplog entries&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-20328&quot;&gt;&lt;del&gt;SERVER-20328&lt;/del&gt;&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="1123908" author="martin.bligh" created="Wed, 30 Dec 2015 13:31:21 +0000"  >&lt;p&gt;The existing design only allows either reads or writes on the secondary at any one time.  Under normal circumstances, these are interleaved by alternating between apply batches on the secondary and allowing reads.&lt;/p&gt;

&lt;p&gt;The current plan to address this involves using a snapshot per application batch on the secondary - we can allow reads from that snapshot whilst the writes progress, without breaking the consistency order that data appears in. However, it is not a small change to make.&lt;/p&gt;</comment>
                            <comment id="1123501" author="fengqu" created="Tue, 29 Dec 2015 18:09:04 +0000"  >&lt;p&gt;what&apos;s plan for fixing this issue? this caused issue we have in CS-23893.&lt;/p&gt;</comment>
                            <comment id="1114746" author="bruce.lucas@10gen.com" created="Tue, 15 Dec 2015 12:33:35 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=zhifan&quot; class=&quot;user-hover&quot; rel=&quot;zhifan&quot;&gt;zhifan&lt;/a&gt;, I meant the latter. It is expected that applying the oplog blocks reads, and this will be especially the case when the secondary is working hard to catch up to the primary and therefore is applying the oplog more-or-less continuously. The question is why does the secondary struggle to keep up with the primary. There are known issues in this area, but the question is whether the particular issue that seems to be hit in this repro, and in your use case, is one of the known issues.&lt;/p&gt;</comment>
                            <comment id="1114395" author="zhifan" created="Tue, 15 Dec 2015 00:54:11 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=Bruce.Lucas&quot; class=&quot;user-hover&quot; rel=&quot;Bruce.Lucas&quot;&gt;Bruce.Lucas&lt;/a&gt;  I didn&apos;t understand you well.  Do you mean why the applying oplog blocked read? or do you mean why the write traffic on primary is ok but the slave can&apos;t catch up with the primary? &lt;/p&gt;</comment>
                            <comment id="1114385" author="david.hows" created="Tue, 15 Dec 2015 00:37:35 +0000"  >&lt;p&gt;Have just run another test with nojournal. In both the 20 thread and 1 thread cases this made no difference in the issue as reported, everything still eventually begins to lag and the read throughput drops off.&lt;/p&gt;</comment>
                            <comment id="1113958" author="bruce.lucas@10gen.com" created="Mon, 14 Dec 2015 19:38:57 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=david.hows&quot; class=&quot;user-hover&quot; rel=&quot;david.hows&quot;&gt;david.hows&lt;/a&gt;, there was a bug (&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-21305&quot; title=&quot;Lock &amp;#39;timeAcquiringMicros&amp;#39; value is much higher than the actual time spent&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-21305&quot;&gt;&lt;del&gt;SERVER-21305&lt;/del&gt;&lt;/a&gt;) in the computation of timeAcquiringMicros that makes the actual number incorrect; fixed in 3.0.8. However you can still use it to determine if there is queuing on the global lock, you just won&apos;t be able to easily determine the length of the queue. &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-21859&quot; title=&quot;The globalLock.currentQueue stat doesn&amp;#39;t include operations queued for the global lock&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-21859&quot;&gt;&lt;del&gt;SERVER-21859&lt;/del&gt;&lt;/a&gt;  means that you can&apos;t use the globalLock.currentQueue stat to determine that, because time spent queueing on every lock but the global lock shows up in globalLock.currentQueue.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=zhifan&quot; class=&quot;user-hover&quot; rel=&quot;zhifan&quot;&gt;zhifan&lt;/a&gt;, you are correct that it is expected that continuous replication workload can starve secondary reads. I think the goal is to determine why, under conditions when it seems there should be excess capacity at the secondary to do secondary reads, it may not be the case because, as it appears, sometimes the secondary struggles to keep up with the primary, as indicated by growing replication lag, at which time secondary reads become starved.&lt;/p&gt;</comment>
                            <comment id="1113253" author="zhifan" created="Mon, 14 Dec 2015 08:05:46 +0000"  >&lt;p&gt;Hi Hows:&lt;br/&gt;
   Why don&apos;t we think it is expected since whenever the slave is applying an oplog, it will block all the reads on slaves?  In your test case, there are continuously update on primary,  so probably there are  continuously update  on slave,  the the read will be blocked (waiting for global lock) in a extremely high possibility. &lt;/p&gt;</comment>
                            <comment id="1113237" author="david.hows" created="Mon, 14 Dec 2015 06:59:56 +0000"  >&lt;p&gt;Just to untangle things and provide a checkpoint as to what I&apos;ve found;&lt;br/&gt;
There are currently two metrics that I have been using to track the issue at hand. I will focus on the single threaded case that I have been working with today and running with MongoDB 3.2;&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;When load is being applied to the primary repl lag is seen of 30+ seconds&lt;/li&gt;
	&lt;li&gt;When load is being applied to the primary, secondary reads will sporadically take very long times to execute.&lt;br/&gt;
&lt;b&gt;Long Operations on Secondary under 3.2&lt;/b&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-14T17:11:49.567+1100 I COMMAND  [conn8] command test.test command: find { find: &quot;test&quot;, filter: { _id: 5865940.0 }, limit: 1.0, singleBatch: true } planSummary: IDHACK keysExamined:1 docsExamined:1 idhack:1 cursorExhausted:1 keyUpdates:0 writeConflicts:0 numYields:0 nreturned:1 reslen:143 locks:{ Global: { acquireCount: { r: 2 }, acquireWaitCount: { r: 1 }, timeAcquiringMicros: { r: 814498 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { r: 1 } } } protocol:op_command 814ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-14T17:11:50.205+1100 I COMMAND  [conn8] command test.test command: find { find: &quot;test&quot;, filter: { _id: 5348929.0 }, limit: 1.0, singleBatch: true } planSummary: IDHACK keysExamined:1 docsExamined:1 idhack:1 cursorExhausted:1 keyUpdates:0 writeConflicts:0 numYields:1 nreturned:1 reslen:143 locks:{ Global: { acquireCount: { r: 4 }, acquireWaitCount: { r: 1 }, timeAcquiringMicros: { r: 1003 } }, Database: { acquireCount: { r: 2 } }, Collection: { acquireCount: { r: 2 } } } protocol:op_command 569ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-14T17:11:51.086+1100 I COMMAND  [conn8] command test.test command: find { find: &quot;test&quot;, filter: { _id: 9562842.0 }, limit: 1.0, singleBatch: true } planSummary: IDHACK keysExamined:1 docsExamined:1 idhack:1 cursorExhausted:1 keyUpdates:0 writeConflicts:0 numYields:1 nreturned:1 reslen:143 locks:{ Global: { acquireCount: { r: 4 }, acquireWaitCount: { r: 1 }, timeAcquiringMicros: { r: 1399 } }, Database: { acquireCount: { r: 2 } }, Collection: { acquireCount: { r: 2 } } } protocol:op_command 785ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-14T17:11:52.624+1100 I COMMAND  [conn8] command test.test command: find { find: &quot;test&quot;, filter: { _id: 6098631.0 }, limit: 1.0, singleBatch: true } planSummary: IDHACK keysExamined:1 docsExamined:1 idhack:1 cursorExhausted:1 keyUpdates:0 writeConflicts:0 numYields:0 nreturned:1 reslen:143 locks:{ Global: { acquireCount: { r: 2 }, acquireWaitCount: { r: 1 }, timeAcquiringMicros: { r: 1530898 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { r: 1 } } } protocol:op_command 1531ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-14T17:11:53.366+1100 I COMMAND  [conn8] command test.test command: find { find: &quot;test&quot;, filter: { _id: 8450642.0 }, limit: 1.0, singleBatch: true } planSummary: IDHACK keysExamined:1 docsExamined:1 idhack:1 cursorExhausted:1 keyUpdates:0 writeConflicts:0 numYields:0 nreturned:1 reslen:143 locks:{ Global: { acquireCount: { r: 2 }, acquireWaitCount: { r: 1 }, timeAcquiringMicros: { r: 741045 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { r: 1 } } } protocol:op_command 741ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-14T17:11:54.843+1100 I COMMAND  [conn8] command test.test command: find { find: &quot;test&quot;, filter: { _id: 3164850.0 }, limit: 1.0, singleBatch: true } planSummary: IDHACK keysExamined:1 docsExamined:1 idhack:1 cursorExhausted:1 keyUpdates:0 writeConflicts:0 numYields:0 nreturned:1 reslen:143 locks:{ Global: { acquireCount: { r: 2 }, acquireWaitCount: { r: 1 }, timeAcquiringMicros: { r: 1472757 } }, Database: { acquireCount: { r: 1 } }, Collection: { acquireCount: { r: 1 } } } protocol:op_command 1472ms&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;The most interesting thing about these operations is that 3/5 show that almost all of operational execution is spent in timeAcquiringMicros. I&apos;m uncertain if the other two would be due to another bug in how timeAcquiringMicros is calculated, if those operations are legitimately taking hundreds of milliseconds for an _id lookup or is a variation on &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-21859&quot; title=&quot;The globalLock.currentQueue stat doesn&amp;#39;t include operations queued for the global lock&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-21859&quot;&gt;&lt;del&gt;SERVER-21859&lt;/del&gt;&lt;/a&gt; or something similar.&lt;/p&gt;

&lt;p&gt;I also tested with the latest MongoDB master and develop and this showed some initial improvements, but over several runs it turns out there was little difference from 3.2. The only noticeable difference was that after a longer run-time the repl lag seems to stabilise under the Master+Develop version.&lt;/p&gt;

&lt;p&gt;Under the single threaded version, the slowdown on the secondaries is very strongly associated with checkpoints in WT (as per the attached ss-singlethread.html), this may also explain the lack of high &lt;tt&gt;timeAcquiringMicros&lt;/tt&gt; on some of the above queries.&lt;/p&gt;

&lt;p&gt;As it stands I&apos;m not certain if this is &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-18908&quot; title=&quot;Secondaries unable to keep up with primary under WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-18908&quot;&gt;&lt;del&gt;SERVER-18908&lt;/del&gt;&lt;/a&gt;/&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-18983&quot; title=&quot;Process oplog inserts, and applying, on the secondary in parallel&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-18983&quot;&gt;&lt;del&gt;SERVER-18983&lt;/del&gt;&lt;/a&gt;, but my suspicions are becoming stronger that &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=bruce.lucas&quot; class=&quot;user-hover&quot; rel=&quot;bruce.lucas&quot;&gt;bruce.lucas&lt;/a&gt; is correct as the high &lt;tt&gt;timeAcquiringMicros&lt;/tt&gt; is for a global lock and consistent with the explanation above.&lt;/p&gt;</comment>
                            <comment id="1113196" author="david.hows" created="Mon, 14 Dec 2015 03:45:57 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=alexander.gorrod&quot; class=&quot;user-hover&quot; rel=&quot;alexander.gorrod&quot;&gt;alexander.gorrod&lt;/a&gt;, I have amended my comment to make my thoughts more clear, I suspect it is less likely that &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-18983&quot; title=&quot;Process oplog inserts, and applying, on the secondary in parallel&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-18983&quot;&gt;&lt;del&gt;SERVER-18983&lt;/del&gt;&lt;/a&gt; is involved as that tickets current focus (as I read it) is on adding a method to apply the oplog on secondaries in parallel. &lt;/p&gt;

&lt;p&gt;Other than the size of the batch jobs that I am working with there should be no other parallelisation within the workload I tested with.&lt;/p&gt;</comment>
                            <comment id="1113192" author="alexander.gorrod" created="Mon, 14 Dec 2015 03:25:19 +0000"  >&lt;p&gt;Thanks for digging deeper &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=david.hows&quot; class=&quot;user-hover&quot; rel=&quot;david.hows&quot;&gt;david.hows&lt;/a&gt; Does the above information make it more or less likely that this is the same problem being tracked by &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-18983&quot; title=&quot;Process oplog inserts, and applying, on the secondary in parallel&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-18983&quot;&gt;&lt;del&gt;SERVER-18983&lt;/del&gt;&lt;/a&gt;?&lt;/p&gt;</comment>
                            <comment id="1113159" author="david.hows" created="Mon, 14 Dec 2015 01:49:14 +0000"  >&lt;p&gt;Modified the script today to do some further diving. Even with only the one thread inserting, I can see times where it takes over 60 seconds to finish the small burst of 10K reads (baseline is between 1-2 seconds) and I can see repl lag spiking greatly during that time.&lt;/p&gt;

&lt;p&gt;It does seem that the system is able to catch up eventually, with at most somewhere near a minutes delay, but this load is one primary receiving one threads worth of update operations only and the secondary still having trouble keeping pace, which means that this may not exclusively be an issue with parallel on primaries vs serial on secondaries.&lt;/p&gt;</comment>
                            <comment id="1111931" author="bruce.lucas@10gen.com" created="Fri, 11 Dec 2015 12:34:06 +0000"  >&lt;p&gt;&lt;span class=&quot;image-wrap&quot; style=&quot;&quot;&gt;&lt;img src=&quot;https://jira.mongodb.org/secure/attachment/101789/101789_lag.png&quot; width=&quot;900&quot; style=&quot;border: 0px solid black&quot; /&gt;&lt;/span&gt;&lt;/p&gt;

&lt;ul&gt;
	&lt;li&gt;Replication lag builds under the insert workload, as indicated by the repl buffer stats. (The 3.2 ftdc data will have a more direct indication of this.) This means replication is struggling, and that is what is probably what is starving the reads, because we &quot;stop the world&quot; using the global lock during replication.&lt;/li&gt;
	&lt;li&gt;This does not show up as read queues because of &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-21859&quot; title=&quot;The globalLock.currentQueue stat doesn&amp;#39;t include operations queued for the global lock&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-21859&quot;&gt;&lt;del&gt;SERVER-21859&lt;/del&gt;&lt;/a&gt;, but the global lock time acquiring stat supports this as you point out. The WT concurrentTransactions metric is also consistent with this - the reads are not queuing in WT.&lt;/li&gt;
	&lt;li&gt;The main operations done by replication are applying the ops and inserting them into the oplog. Time spent applying the ops is reported in &quot;repl apply batches totalMillis&quot;, but this is not large, so the replication lag is most likely related to inserting ops into the oplog, and that is also consistent with the customer&apos;s observation.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;If the above is correct, this is a known issue, and there are a couple tickets tracking work in this area - &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-18908&quot; title=&quot;Secondaries unable to keep up with primary under WiredTiger&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-18908&quot;&gt;&lt;del&gt;SERVER-18908&lt;/del&gt;&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-18983&quot; title=&quot;Process oplog inserts, and applying, on the secondary in parallel&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-18983&quot;&gt;&lt;del&gt;SERVER-18983&lt;/del&gt;&lt;/a&gt;. In this case for example you have 20 threads on the primary inserting into the oplog in parallel whereas that is serialized on the secondary, so that may account for the lag. In any case stack traces would help diagnose this of course.&lt;/p&gt;</comment>
                            <comment id="1111737" author="david.hows" created="Fri, 11 Dec 2015 05:39:17 +0000"  >&lt;p&gt;Current working hypothesis is that there is some delay in the processing of replication operations that is causing the delay.&lt;/p&gt;

&lt;p&gt;The aim with the reproduction workload was to generate a large number of deletes at once when truncating the oplog to allow for the insertion of a new very large document in an update only workload.&lt;/p&gt;

&lt;p&gt;Initially, I suspected this may have been related to tombstones within the oplog collection in WiredTiger, thus the large/small update workload in a replica set. However, as &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-21027&quot; title=&quot;Reduced performance of index lookups after removing documents from collection&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-21027&quot;&gt;&lt;del&gt;SERVER-21027&lt;/del&gt;&lt;/a&gt; is fixed within 3.2.0, I am uncertain if this is the cause.&lt;/p&gt;
</comment>
                            <comment id="1111716" author="david.hows" created="Fri, 11 Dec 2015 05:18:26 +0000"  >&lt;p&gt;Attaching the results of timeseries on the serverStatus captures on 2 repro runs.&lt;/p&gt;

&lt;p&gt;The 3.0.7 run is far longer as it was the initial reproduction and some changes were made to reproducer to get things going. You can also see a point when I stopped the reproduction to confirm that execution eventually returned to normal (it did).&lt;/p&gt;

&lt;p&gt;The 3.2.0 run is far more indicative, as it was just load and reproduce.&lt;/p&gt;

&lt;p&gt;Probably the most telling symptom at this stage is the &lt;tt&gt;Global timeAcquiringMicros r&lt;/tt&gt; in the 3.2.0 timeseries attachment, it begins spiking as soon as the ability to read falls behind.&lt;/p&gt;</comment>
                            <comment id="1111708" author="david.hows" created="Fri, 11 Dec 2015 05:08:55 +0000"  >&lt;p&gt;The repro script requires the MongoDB ruby driver v2.2.0.&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                            <outwardlinks description="duplicates">
                                        <issuelink>
            <issuekey id="228962">SERVER-20328</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                                        </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="209769">SERVER-18908</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="210754">SERVER-18983</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="439084">SERVER-31359</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="101737" name="307.html" size="5136035" author="david.hows" created="Fri, 11 Dec 2015 05:18:26 +0000"/>
                            <attachment id="101738" name="320.html" size="5133322" author="david.hows" created="Fri, 11 Dec 2015 05:18:26 +0000"/>
                            <attachment id="101789" name="lag.png" size="256979" author="bruce.lucas@mongodb.com" created="Fri, 11 Dec 2015 12:29:42 +0000"/>
                            <attachment id="101734" name="repro.rb" size="902" author="david.hows" created="Fri, 11 Dec 2015 05:08:55 +0000"/>
                            <attachment id="102080" name="ss-singlethread.html" size="5271810" author="david.hows" created="Mon, 14 Dec 2015 07:00:17 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>16.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                <customfield id="customfield_12751" key="com.atlassian.jira.plugin.system.customfieldtypes:multiselect">
                        <customfieldname>Assigned Teams</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="25136"><![CDATA[Storage Execution]]></customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                            <customfield id="customfield_10011" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Backwards Compatibility</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10038"><![CDATA[Fully Compatible]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 11 Dec 2015 05:21:56 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        6 years, 7 weeks, 6 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>alexander.golin@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            6 years, 7 weeks, 6 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>alexander.gorrod@mongodb.com</customfieldvalue>
            <customfieldvalue>asya.kamsky@mongodb.com</customfieldvalue>
            <customfieldvalue>zhifan</customfieldvalue>
            <customfieldvalue>backlog-server-execution</customfieldvalue>
            <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>david.hows</customfieldvalue>
            <customfieldvalue>fengqu</customfieldvalue>
            <customfieldvalue>martin.bligh</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrkmg7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hr9rmn:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10750" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Steps To Reproduce</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>&lt;p&gt;Launch mongod&apos;s:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;./mongod --replSet foo --storageEngine wiredTiger --dbpath /mnt/work/data/rs0 --oplogSize 1000 --wiredTigerCacheSizeGB 4 --logpath /mnt/work/data/rs0/mongod.log --fork&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;./mongod --replSet foo --storageEngine wiredTiger --dbpath /mnt/work/data/rs1 --oplogSize 1000 --wiredTigerCacheSizeGB 4 --port 27018 --logpath /mnt/work/data/rs1/mongod.log --fork&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Setup the replset:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;rs.initiate()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;rs.add(&quot;localhost:27018&quot;)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Run javascript against the primary to seed documents&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;var INT_MAX=10000000;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;for(x=0;x&amp;lt;INT_MAX/1000;x++){&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	var bulk = db.test.initializeUnorderedBulkOp()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	for(i=0;i&amp;lt;1000;i++){&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;		var num = (x*1000)+i;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;		bulk.insert({_id:num,x:num,y:num});&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	bulk.execute();&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Execute the attached &lt;tt&gt;repro.rb&lt;/tt&gt;&lt;/p&gt;

&lt;p&gt;Run the following javascript on the secondary to monitor performance. You should see an initially stable number of query times, which will then blow out to seconds and minutes:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;while(true){&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;var INT_MAX=10000000;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;start = new Date()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;rs.slaveOk()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;for(i=0;i&amp;lt;10000;i++){&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	var docNum = Math.floor(Math.random()*INT_MAX);&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	db.test.findOne({_id:docNum});&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;print(&quot;Took: &quot; +(new Date() - start) + &quot;ms&quot;)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;sleep(1000);&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrv3c7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>