<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:52:49 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-19997] Secondary crashes with oplog stream went back in time</title>
                <link>https://jira.mongodb.org/browse/SERVER-19997</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;We ran into a crash on 3.0.5 where the oplog was reported as going back in time, but the timestamps in the log match&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;replication oplog stream went back in time. previous timestamp: 55d25074:41 newest timestamp: 55d25074:41. Op being applied: { ts: Timestamp 1439846516000|65, h: 1280351009425609402, v: 2, op: &quot;i&quot;, ns: &quot;...&quot;, o: { ... }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</description>
                <environment></environment>
        <key id="226036">SERVER-19997</key>
            <summary>Secondary crashes with oplog stream went back in time</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="4">Incomplete</resolution>
                                        <assignee username="milkie@mongodb.com">Eric Milkie</assignee>
                                    <reporter username="dynamike">Michael Kania</reporter>
                        <labels>
                    </labels>
                <created>Tue, 18 Aug 2015 00:06:11 +0000</created>
                <updated>Tue, 9 Aug 2016 13:06:55 +0000</updated>
                            <resolved>Fri, 19 Feb 2016 21:03:18 +0000</resolved>
                                    <version>3.0.5</version>
                                                    <component>Replication</component>
                                        <votes>1</votes>
                                    <watches>15</watches>
                                                                                                                <comments>
                            <comment id="1116311" author="ramon.fernandez" created="Wed, 16 Dec 2015 18:43:49 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=udit&quot; class=&quot;user-hover&quot; rel=&quot;udit&quot;&gt;udit&lt;/a&gt;, sorry you&apos;re running into this issue. If this behavior persists after upgrading to 3.2.0, can you please open a separate ticket so we can investigate your issue separately?&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="1116200" author="udit" created="Wed, 16 Dec 2015 16:48:56 +0000"  >&lt;p&gt;We are experiencing it very frequent (every 2-3 hours) on secondary nodes.&lt;/p&gt;

&lt;p&gt;Logs:&lt;/p&gt;

&lt;p&gt;2015-12-16T10:41:56.052-0500 F REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsSync&amp;#93;&lt;/span&gt; replication oplog stream went back in time. previous timestamp: 56718779:76c newest timestamp: 56718704:536. Op being applied&lt;br/&gt;
2015-12-16T10:41:56.052-0500 I -        &lt;span class=&quot;error&quot;&gt;&amp;#91;rsSync&amp;#93;&lt;/span&gt; Fatal Assertion 18905&lt;br/&gt;
2015-12-16T10:41:56.052-0500 I -        &lt;span class=&quot;error&quot;&gt;&amp;#91;rsSync&amp;#93;&lt;/span&gt; &lt;/p&gt;

&lt;p&gt;Environment :&lt;/p&gt;

&lt;p&gt;OS : Ubuntu 14.04&lt;br/&gt;
Mongo DB version : 3.0.8&lt;br/&gt;
Storage Engine  : Wired Tiger&lt;br/&gt;
AWS instance : I2.xlarge (using local SSD storage of xfs fils system)&lt;/p&gt;

&lt;p&gt;As a next step, we are upgrading the replica set to 3.2.0 to see if it fix it.&lt;/p&gt;


</comment>
                            <comment id="1109180" author="ma6174" created="Wed, 9 Dec 2015 09:11:08 +0000"  >&lt;p&gt;I have the similar crash:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-09T06:30:46.903+0800 E REPL     [rsBackgroundSync] sync producer problem: 16235 going to start syncing, but buffer is not empty&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-09T06:30:47.260+0800 I REPL     [ReplicationExecutor] syncing from: node_slave1:2701&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-09T06:30:47.399+0800 I REPL     [SyncSourceFeedback] replset setting syncSourceFeedback to node_slave1:2701&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-09T06:30:47.604+0800 F REPL     [rsSync] replication oplog stream went back in time. previous timestamp: 56675270:239 newest timestamp: 5667526f:653. Op being applied: { ts: Timestamp 1449611887000|1619, h: 7253060569348003209, v: 2, op: &quot;u&quot;, ns: &quot;db1.coll1&quot;, o2: { _id: ... }, o: { ... } }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-09T06:30:47.605+0800 I -        [rsSync] Fatal Assertion 18905&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-09T06:30:47.605+0800 I -        [rsSync]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;***aborting after fassert() failure&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;what&apos;s different:&lt;/p&gt;

&lt;p&gt;1. All node version is 3.0.5&lt;br/&gt;
2. after crash, we restart the slave, it not crash again&lt;/p&gt;

&lt;p&gt;there are some logs before crash&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-09T06:30:00.243+0800 I NETWORK  [rsBackgroundSync] Socket recv() timeout  node_slave1:2701&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-09T06:30:00.273+0800 I NETWORK  [rsBackgroundSync] SocketException: remote: node_slave1:2701 error: 9001 socket exception [RECV_TIMEOUT] server [node_slave1:2701]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-09T06:30:00.333+0800 E REPL     [rsBackgroundSync] sync producer problem: 10278 dbclient error communicating with server: node_slave1:2701&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-09T06:30:00.357+0800 I REPL     [ReplicationExecutor] syncing from: node_slave1:2701&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-09T06:30:00.397+0800 I NETWORK  [conn832] end connection node_slave1:36844 (53 connections now open)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-09T06:30:00.397+0800 I NETWORK  [initandlisten] connection accepted from node_slave1:37024 #837 (54 connections now open)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-09T06:30:00.874+0800 I REPL     [SyncSourceFeedback] replset setting syncSourceFeedback to node_slave1:2701&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-09T06:30:03.619+0800 I REPL     [rsBackgroundSync] replSet syncTail condition 1&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-09T06:30:04.619+0800 I -        [rsBackgroundSync] Assertion: 16235:going to start syncing, but buffer is not empty&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;2015-12-09T06:30:04.710+0800 I CONTROL  [rsBackgroundSync]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; 0xf77cc9 0xf16b81 0xefbddf 0xc15d20 0xc173b4 0xc17458 0xfc4e84 0x7f756be01e9a 0x7f756a8d838d&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;----- BEGIN BACKTRACE -----&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{&quot;backtrace&quot;:[{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;B77CC9&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;B16B81&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;AFBDDF&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;815D20&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;8173B4&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;817458&quot;},{&quot;b&quot;:&quot;400000&quot;,&quot;o&quot;:&quot;BC4E84&quot;},{&quot;b&quot;:&quot;7F756BDFA000&quot;,&quot;o&quot;:&quot;7E9A&quot;},{&quot;b&quot;:&quot;7F756A7E5000&quot;,&quot;o&quot;:&quot;F338D&quot;}],&quot;processInfo&quot;:{ &quot;mongodbVersion&quot; : &quot;3.0.5&quot;, &quot;gitVersion&quot; : &quot;8bc4ae20708dbb493cb09338d9e7be6698e4a3a3&quot;, &quot;uname&quot; : { &quot;sysname&quot; : &quot;Linux&quot;, &quot;release&quot; : &quot;3.2.0-23-generic&quot;, &quot;version&quot; : &quot;#36-Ubuntu SMP Tue Apr 10 20:39:51 UTC 2012&quot;, &quot;machine&quot; : &quot;x86_64&quot; }, &quot;somap&quot; : [ { &quot;elfType&quot; : 2, &quot;b&quot; : &quot;400000&quot;, &quot;buildId&quot; : &quot;1D3BB52AB05E62C119D95718E82CD0447A8CE0FA&quot; }, { &quot;b&quot; : &quot;7FFFE72FF000&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;0C0CC7757C6933FC35A55C90475D300ADAF970C5&quot; }, { &quot;b&quot; : &quot;7F756BDFA000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libpthread.so.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;C340AF9DEE97C17C730F7D03693286C5194A46B8&quot; }, { &quot;b&quot; : &quot;7F756BB9C000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libssl.so.1.0.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;425A091D0C88F36982CD69122BDB853F6E261DA0&quot; }, { &quot;b&quot; : &quot;7F756B7C1000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libcrypto.so.1.0.0&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;EB6660D716414E4EFB6C9E74657DDD709CB2383F&quot; }, { &quot;b&quot; : &quot;7F756B5B9000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/librt.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;352C5B373A50E6C4AB881A5DB6F5766FDF81EEE0&quot; }, { &quot;b&quot; : &quot;7F756B3B5000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libdl.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;D181AF551DBBC43E9D55913D532635FDE18E7C4E&quot; }, { &quot;b&quot; : &quot;7F756B0B5000&quot;, &quot;path&quot; : &quot;/usr/lib/x86_64-linux-gnu/libstdc++.so.6&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;B534DA725D06A04267EB2FEB92B9CC14C838B57B&quot; }, { &quot;b&quot; : &quot;7F756ADB9000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libm.so.6&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;3CD696AF1C2A66D38B949CB0D1D2A218B3ED858B&quot; }, { &quot;b&quot; : &quot;7F756ABA3000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libgcc_s.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;ECF322A96E26633C5D10F18215170DD4395AF82C&quot; }, { &quot;b&quot; : &quot;7F756A7E5000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libc.so.6&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;AA64A66AC46BFF200848C0A0694011BD0140AB4E&quot; }, { &quot;b&quot; : &quot;7F756C017000&quot;, &quot;path&quot; : &quot;/lib64/ld-linux-x86-64.so.2&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;E25AD1A11CCF57E734116B8EC9C69F643DCA9F18&quot; }, { &quot;b&quot; : &quot;7F756A5CE000&quot;, &quot;path&quot; : &quot;/lib/x86_64-linux-gnu/libz.so.1&quot;, &quot;elfType&quot; : 3, &quot;buildId&quot; : &quot;F695ECFCF3918D5D34989398A14B7ECDD9F46CD0&quot; } ] }}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf77cc9]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf16b81]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xefbddf]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo4repl14BackgroundSync5startEPNS_16OperationContextE+0x320) [0xc15d20]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo4repl14BackgroundSync15_producerThreadEv+0xA4) [0xc173b4]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(_ZN5mongo4repl14BackgroundSync14producerThreadEv+0x68) [0xc17458]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; mongod(+0xBC4E84) [0xfc4e84]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; libpthread.so.0(+0x7E9A) [0x7f756be01e9a]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; libc.so.6(clone+0x6D) [0x7f756a8d838d]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;-----  END BACKTRACE  -----&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="1088172" author="ramon.fernandez" created="Fri, 13 Nov 2015 19:19:15 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=tredman%40fb.com&quot; class=&quot;user-hover&quot; rel=&quot;tredman@fb.com&quot;&gt;tredman@fb.com&lt;/a&gt;, &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=dynamike&quot; class=&quot;user-hover&quot; rel=&quot;dynamike&quot;&gt;dynamike&lt;/a&gt;, is this still an issue for you? Does Eric&apos;s last message address your questions? Please let us know.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Ram&#243;n.&lt;/p&gt;</comment>
                            <comment id="1027290" author="milkie" created="Fri, 11 Sep 2015 11:27:20 +0000"  >&lt;p&gt;Hi Travis,&lt;br/&gt;
The new instances you have posted appear to be a different problem.  The original problem was due to a duplicated op in an oplog on a 2.6 primary node.  For your new instances, the error message on the secondaries indicates to me that it isn&apos;t a duplicated op; instead, it would appear that the secondary nodes&apos; query on the sync source&apos;s oplog is not returning data that matches up with a previous oplog query.&lt;br/&gt;
In the new log snippits, I see that just prior to querying a sync source&apos;s oplog, a new connection is made.  If that connection is to a different node than the one that was used to copy the documents, this problem may have a chance of occurring.  I can&apos;t tell more without the replica set configuration and the full logs from each of the nodes (the version of mongodb that is running on each node is important, as well as each node&apos;s initial sync source, which appears earlier in the logs than what you have initially provided here).&lt;/p&gt;</comment>
                            <comment id="1026911" author="tredman@fb.com" created="Thu, 10 Sep 2015 22:59:39 +0000"  >&lt;p&gt;Hi Eric,&lt;/p&gt;

&lt;p&gt;Sorry, I wasn&apos;t able to get an answer to your question because I didn&apos;t have log data going back that far. Pinging the task today because we had another incident today. We had a replica set with three new nodes doing initial sync, and all three crashed within minutes of each other with this same issue.&lt;/p&gt;

&lt;p&gt;The conditions were slightly different than our previous report because the timestamp difference was very large. In this first example, there was a 84762 second difference between the the current op and the next op.&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:09.514 I INDEX    [rsSync] build index on: appdata100.app_57d46c89-3aaf-4817-82ff-e6ac6404d106:_Index properties: { v: 1, key: { _id: 1 }, name: &quot;_id_&quot;, ns: &quot;appdata100.app_57d46c89-3aaf-4817-82ff-e6ac6404d106:_Index&quot; }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:09.514 I INDEX    [rsSync]      building index using bulk method&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:09.514 D STORAGE  [rsSync] appdata100.app_57d46c89-3aaf-4817-82ff-e6ac6404d106:_Index: clearing plan cache - collection info cache reset&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:09.514 D INDEX    [rsSync]      bulk commit starting for index: _id_&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:09.514 D INDEX    [rsSync]      done building bottom layer, going to commit&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:09.514 I INDEX    [rsSync] build index done.  scanned 0 total records. 0 secs&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:09.514 D STORAGE  [rsSync] looking up metadata for: appdata100.app_57d46c89-3aaf-4817-82ff-e6ac6404d106:_Index @ RecordId(396770)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:09.514 D STORAGE  [rsSync] looking up metadata for: appdata100.app_57d46c89-3aaf-4817-82ff-e6ac6404d106:_Index @ RecordId(396770)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:09.514 D STORAGE  [rsSync] appdata100.app_57d46c89-3aaf-4817-82ff-e6ac6404d106:_Index: clearing plan cache - collection info cache reset&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:09.514 D STORAGE  [rsSync] appdata100.app_57d46c89-3aaf-4817-82ff-e6ac6404d106:_Index: clearing plan cache - collection info cache reset&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:09.516 I REPL     [rsSync] initial sync cloning db: config&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:09.519 D NETWORK  [rsSync] connected to server db104:27017 (10.252.5.205)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:09.554 I REPL     [rsSync] initial sync data copy, starting syncup&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:09.848 I REPL     [rsSync] oplog sync 1 of 3&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:09.885 F REPL     [rsSync] replication oplog stream went back in time. previous timestamp: 55f15435:c4 newest timestamp: 55f0091b:6f. Op being applied: { ts: Timestamp 1441794331000|111, h: -3122181208205084538, v: 2, op: &quot;u&quot;, ns: &quot;appdata119.app_15acc0d6-faf3-4681-a963-f91ec5f7d588:Locations&quot;, o2: { _id: &quot;Exiw3lvQZY&quot; }, o: { $set: { _updated_at: new Date(1441794331548), rankValue: -7.445298078517805e-08 } } }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:09.885 I -        [rsSync] Fatal Assertion 18905&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:09.885 I -        [rsSync]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;In this second example, the difference was 84651 seconds. Note that the timestamps are different in each case, as is the op being applied at the time of the crash.&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:10.229 I INDEX    [rsSync] build index on: appdata100.app_57d46c89-3aaf-4817-82ff-e6ac6404d106:_Index properties: { v: 1, key: { _id: 1 }, name: &quot;_id_&quot;, ns: &quot;appdata100.app_57d46c89-3aaf-4817-82ff-e6ac6404d106:_Index&quot; }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:10.229 I INDEX    [rsSync]      building index using bulk method&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:10.229 D STORAGE  [rsSync] appdata100.app_57d46c89-3aaf-4817-82ff-e6ac6404d106:_Index: clearing plan cache - collection info cache reset&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:10.229 D INDEX    [rsSync]      bulk commit starting for index: _id_&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:10.229 D INDEX    [rsSync]      done building bottom layer, going to commit&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:10.229 I INDEX    [rsSync] build index done.  scanned 0 total records. 0 secs&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:10.229 D STORAGE  [rsSync] looking up metadata for: appdata100.app_57d46c89-3aaf-4817-82ff-e6ac6404d106:_Index @ RecordId(396770)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:10.229 D STORAGE  [rsSync] looking up metadata for: appdata100.app_57d46c89-3aaf-4817-82ff-e6ac6404d106:_Index @ RecordId(396770)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:10.229 D STORAGE  [rsSync] appdata100.app_57d46c89-3aaf-4817-82ff-e6ac6404d106:_Index: clearing plan cache - collection info cache reset&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:10.229 D STORAGE  [rsSync] appdata100.app_57d46c89-3aaf-4817-82ff-e6ac6404d106:_Index: clearing plan cache - collection info cache reset&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:10.230 I REPL     [rsSync] initial sync cloning db: config&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:10.233 D NETWORK  [rsSync] connected to server db104:27017 (10.252.5.205)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:10.254 I REPL     [rsSync] initial sync data copy, starting syncup&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:10.345 I REPL     [rsSync] oplog sync 1 of 3&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:10.361 F REPL     [rsSync] replication oplog stream went back in time. previous timestamp: 55f153d6:125 newest timestamp: 55f0092b:7d. Op being applied: { ts: Timestamp 1441794347000|125, h: 7670102112123622001, v: 2, op: &quot;u&quot;, ns: &quot;appdata111.app_26044e27-cfbd-4cf2-8ab1-9a9228b3b2bb:_Installation&quot;, o2: { _id: &quot;N8vdqzy1LJ&quot; }, o: { $set: { _updated_at: new Date(1441794347499) } } }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:10.361 I -        [rsSync] Fatal Assertion 18905&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Thu Sep 10 20:39:10.361 I -        [rsSync]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;These nodes had been undergoing initial sync for about 48H. We&apos;ve restarted initial sync and will see how it goes this time, but I wonder if there is anything we can do to debug this more proactively/efficiently? Waiting days to repro the issue makes this hard to track down. I&apos;ll happily set up some extra data collection or something during the initial sync if it will help.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Travis&lt;/p&gt;</comment>
                            <comment id="1014883" author="milkie" created="Wed, 26 Aug 2015 14:18:47 +0000"  >&lt;p&gt;Thanks Travis.  That op was recorded on Tue, 18 Aug 2015 at 22:42:36, immediately after a reconfig was run.  Do you know if the node with the duplicate op was primary at 22:42:36?  If you have all the logs from Tuesday, we can check.  I see that the node with the duplicate op is running version 2.6.7; the latest version of the 2.6 branch is 2.6.11.  I will go through the fixes in later versions to see if something related to this symptom has been reported and fixed.&lt;/p&gt;</comment>
                            <comment id="1012096" author="tredman@fb.com" created="Fri, 21 Aug 2015 19:29:56 +0000"  >&lt;p&gt;Build info for the primary.&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongodata3:PRIMARY&amp;gt; db.serverBuildInfo()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        &quot;version&quot; : &quot;2.6.7&quot;,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        &quot;gitVersion&quot; : &quot;a7d57ad27c382de82e9cb93bf983a80fd9ac9899&quot;,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        &quot;OpenSSLVersion&quot; : &quot;&quot;,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        &quot;sysInfo&quot; : &quot;Linux build7.nj1.10gen.cc 2.6.32-431.3.1.el6.x86_64 #1 SMP Fri Jan 3 21:39:27 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49&quot;,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        &quot;loaderFlags&quot; : &quot;-fPIC -pthread -Wl,-z,now -rdynamic&quot;,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        &quot;compilerFlags&quot; : &quot;-Wnon-virtual-dtor -Woverloaded-virtual -fPIC -fno-strict-aliasing -ggdb -pthread -Wall -Wsign-compare -Wno-unknown-pragmas -Winvalid-pch -pipe -Werror -O3 -Wno-unused-function -Wno-deprecated-declarations -fno-builtin-memcmp&quot;,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        &quot;allocator&quot; : &quot;tcmalloc&quot;,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        &quot;versionArray&quot; : [&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;                2,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;                6,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;                7,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;                0&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        ],&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        &quot;javascriptEngine&quot; : &quot;V8&quot;,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        &quot;bits&quot; : 64,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        &quot;debug&quot; : false,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        &quot;maxBsonObjectSize&quot; : 16777216,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;        &quot;ok&quot; : 1&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="1012087" author="tredman@fb.com" created="Fri, 21 Aug 2015 19:20:29 +0000"  >&lt;p&gt;Hi Eric, Mike is on leave for the next few weeks so I&apos;m going to be managing this ticket from our end. &lt;/p&gt;

&lt;p&gt;I ran the above query on the primary and it does look like there are multiple entries for the same op:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;mongodata3:PRIMARY&amp;gt; db.oplog.rs.find( {ts: {$gte:Timestamp(1439937756,69) }}, {}, 5,0,5,8 )&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{ &quot;ts&quot; : Timestamp(1439937756, 69), &quot;h&quot; : NumberLong(&quot;-5240046610690244083&quot;), &quot;v&quot; : 2, &quot;op&quot; : &quot;n&quot;, &quot;ns&quot; : &quot;&quot;, &quot;o&quot; : { &quot;msg&quot; : &quot;Reconfig set&quot;, &quot;version&quot; : 83 } }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{ &quot;ts&quot; : Timestamp(1439937756, 70), &quot;h&quot; : NumberLong(&quot;1660986903740963784&quot;), &quot;v&quot; : 2, &quot;op&quot; : &quot;i&quot;, &quot;ns&quot; : &quot;ABC:XXX&quot;, &quot;o&quot; : { &quot;_id&quot; : &quot;bWsEVm2o1J&quot;, &quot;IP&quot; : &quot;X.X.X.X&quot;, &quot;Serial&quot; : &quot;app&quot;, &quot;OS&quot; : &quot;5.0.1&quot;, &quot;Locale&quot; : &quot;&quot;, &quot;FirstName&quot; : &quot;XXX&quot;, &quot;LoyaltyPoints&quot; : &quot;5&quot;, &quot;_rperm&quot; : [ &quot;xEqWRGEwoD&quot; ], &quot;LastName&quot; : &quot;&quot;, &quot;Rank&quot; : &quot;5&quot;, &quot;_wperm&quot; : [ &quot;xEqWRGEwoD&quot; ], &quot;DeviceId&quot; : &quot;357197051399544&quot;, &quot;DevicePlatform&quot; : &quot;Android&quot;, &quot;DeviceType&quot; : &quot;Phone&quot;, &quot;SCNMemberKey&quot; : &quot;&quot;, &quot;LoyaltyStatusId&quot; : &quot;10&quot;, &quot;PointsDaily&quot; : &quot;7949&quot;, &quot;PointsWeekly&quot; : &quot;7949&quot;, &quot;Points&quot; : &quot;7949&quot;, &quot;AppVersion&quot; : &quot;22.65.0&quot;, &quot;Visits&quot; : &quot;4&quot;, &quot;IsPayingCustomer&quot; : &quot;true&quot;, &quot;Credits&quot; : &quot;15616&quot;, &quot;_updated_at&quot; : ISODate(&quot;2015-08-18T22:42:36.424Z&quot;), &quot;_acl&quot; : { &quot;xEqWRGEwoD&quot; : { &quot;r&quot; : true, &quot;w&quot; : true } }, &quot;PurchasedPremiumCount&quot; : &quot;0&quot;, &quot;_created_at&quot; : ISODate(&quot;2015-08-18T22:42:36.424Z&quot;), &quot;UserId&quot; : &quot;10700667&quot; } }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{ &quot;ts&quot; : Timestamp(1439937756, 70), &quot;h&quot; : NumberLong(&quot;1660986903740963784&quot;), &quot;v&quot; : 2, &quot;op&quot; : &quot;i&quot;, &quot;ns&quot; : &quot;ABC:XXX&quot;, &quot;o&quot; : { &quot;_id&quot; : &quot;bWsEVm2o1J&quot;, &quot;IP&quot; : &quot;X.X.X.X&quot;, &quot;Serial&quot; : &quot;app&quot;, &quot;OS&quot; : &quot;5.0.1&quot;, &quot;Locale&quot; : &quot;&quot;, &quot;FirstName&quot; : &quot;XXX&quot;, &quot;LoyaltyPoints&quot; : &quot;5&quot;, &quot;_rperm&quot; : [ &quot;xEqWRGEwoD&quot; ], &quot;LastName&quot; : &quot;&quot;, &quot;Rank&quot; : &quot;5&quot;, &quot;_wperm&quot; : [ &quot;xEqWRGEwoD&quot; ], &quot;DeviceId&quot; : &quot;357197051399544&quot;, &quot;DevicePlatform&quot; : &quot;Android&quot;, &quot;DeviceType&quot; : &quot;Phone&quot;, &quot;SCNMemberKey&quot; : &quot;&quot;, &quot;LoyaltyStatusId&quot; : &quot;10&quot;, &quot;PointsDaily&quot; : &quot;7949&quot;, &quot;PointsWeekly&quot; : &quot;7949&quot;, &quot;Points&quot; : &quot;7949&quot;, &quot;AppVersion&quot; : &quot;22.65.0&quot;, &quot;Visits&quot; : &quot;4&quot;, &quot;IsPayingCustomer&quot; : &quot;true&quot;, &quot;Credits&quot; : &quot;15616&quot;, &quot;_updated_at&quot; : ISODate(&quot;2015-08-18T22:42:36.424Z&quot;), &quot;_acl&quot; : { &quot;xEqWRGEwoD&quot; : { &quot;r&quot; : true, &quot;w&quot; : true } }, &quot;PurchasedPremiumCount&quot; : &quot;0&quot;, &quot;_created_at&quot; : ISODate(&quot;2015-08-18T22:42:36.424Z&quot;), &quot;UserId&quot; : &quot;10700667&quot; } }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{ &quot;ts&quot; : Timestamp(1439937782, 1), &quot;h&quot; : NumberLong(&quot;4124812040359931130&quot;), &quot;v&quot; : 2, &quot;op&quot; : &quot;u&quot;, &quot;ns&quot; : &quot;ABC:XXX&quot;, &quot;o2&quot; : { &quot;_id&quot; : &quot;FjuzozeNAk&quot; }, &quot;o&quot; : { &quot;$set&quot; : { &quot;message&quot; : &quot;{\&quot;message\&quot;:\&quot;TIME_OK\&quot;,\&quot;status\&quot;:{},\&quot;totalApiRequests\&quot;:28,\&quot;timeStarted\&quot;:1439937726180,\&quot;lateStartSeconds\&quot;:6.986,\&quot;timeElapsedMills\&quot;:21918,\&quot;timeElapsedMessage\&quot;:\&quot;21.918s\&quot;,\&quot;name\&quot;:\&quot;MASTER\&quot;,\&quot;timeSinceSeasonEndProcessRun\&quot;:1439937748098,\&quot;timeSeasonProcessingStartedAt\&quot;:0,\&quot;PROCESS_REJECTED_INVITATIONS\&quot;:true,\&quot;PROCESS_EXPIRED_INVITATIONS\&quot;:true,\&quot;PROCESS_SUCCESSFUL_INVITATIONS\&quot;:true,\&quot;PROCESS_REQUESTED_INVITATIONS\&quot;:true,\&quot;PROCESS_FINISHED_MATCHES\&quot;:true,\&quot;PROCESS_PVP_REQUESTS\&quot;:true,\&quot;PROCESS_TRANSFERS\&quot;:true,\&quot;PROCESS_RENAMED_TEAMS\&quot;:true,\&quot;CLEAR_INACTIVE_TEAMS\&quot;:true,\&quot;PROCESS_PURCHASES\&quot;:true,\&quot;PROCESS_CHECK_SEASON_END\&quot;:true,\&quot;MAX_RUN_TIME\&quot;:170000,\&quot;totalRuns\&quot;:3,\&quot;runsAndTimes\&quot;:[\&quot;1:852\&quot;,\&quot;2:10862\&quot;,\&quot;3:21011\&quot;],\&quot;terminationTime\&quot;:0,\&quot;terminationAt\&quot;:\&quot;\&quot;,\&quot;day\&quot;:9,\&quot;cancellationPromise\&quot;:{\&quot;_resolved\&quot;:false,\&quot;_rejected\&quot;:false,\&quot;_resolvedCallbacks\&quot;:[null],\&quot;_rejectedCallbacks\&quot;:[null]},\&quot;FORCE_TEAMS_CREATION\&quot;:false}&quot; } } }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{ &quot;ts&quot; : Timestamp(1439937782, 2), &quot;h&quot; : NumberLong(&quot;-6066342674646170879&quot;), &quot;v&quot; : 2, &quot;op&quot; : &quot;u&quot;, &quot;ns&quot; : &quot;ABC:XXX&quot;, &quot;o2&quot; : { &quot;_id&quot; : &quot;XdF2rYZ9Mu&quot; }, &quot;o&quot; : { &quot;$set&quot; : { &quot;sessions&quot; : NumberLong(1262), &quot;lastSession&quot; : ISODate(&quot;2015-08-18T22:42:36.520Z&quot;), &quot;pageViews&quot; : NumberLong(3625), &quot;_updated_at&quot; : ISODate(&quot;2015-08-18T22:42:36.744Z&quot;) } } }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                            <comment id="1011195" author="milkie" created="Thu, 20 Aug 2015 20:45:55 +0000"  >&lt;p&gt;Michael, the file you attached &quot;md3-db2-full.log.tar.gz&quot; is the same as the file you attached earlier &quot;md3-db1.gz&quot;; I suspect something went wrong with the generation of the full log?&lt;/p&gt;</comment>
                            <comment id="1010752" author="milkie" created="Thu, 20 Aug 2015 14:54:49 +0000"  >&lt;p&gt;Since the node fails to restart with the same error, the problem may be on the source rather than the target node.  Can you query the oplog on all other nodes and confirm there is only one document with &lt;/p&gt;
{ &quot;ts&quot; : Timestamp(1439937756, 70), &quot;h&quot; : NumberLong(&quot;1660986903740963784&quot;) }
&lt;p&gt;?&lt;br/&gt;
Something like this should work:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;use local&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;rs.slaveOk()&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;db.oplog.rs.find( {ts: {$gte:Timestamp(1439937756,69) }}, {}, 5,0,5,8 )&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;The numbers (5,0,5,8) are, respectively, limit, skip, batchSize, and options (a bitmap) &amp;#8211; &quot;8&quot; means use OplogReplay which enables the query to use a special optimized algorithm to return results.  By using OplogReplay, the projection field of the find() function is ignored so I took it out.&lt;/p&gt;

&lt;p&gt;If the problem is indeed on the source, rebuilding the node will appear to help because the target will only look at the oplog on the source past the point of the problem.&lt;/p&gt;</comment>
                            <comment id="1010266" author="dynamike" created="Thu, 20 Aug 2015 00:02:37 +0000"  >&lt;p&gt;Attached a larger log for m3-db2. &lt;/p&gt;</comment>
                            <comment id="1010240" author="dynamike" created="Wed, 19 Aug 2015 23:15:29 +0000"  >&lt;p&gt;The node fails to restart with the same error so I&apos;m left with having to rebuild the node&lt;/p&gt;</comment>
                            <comment id="1010096" author="milkie" created="Wed, 19 Aug 2015 21:17:41 +0000"  >&lt;p&gt;Do you have further history for the log for md3-db2?  I was hoping to determine what node it was syncing from.  This would be the node that fed it an incorrect op after the election of md3-db1.&lt;/p&gt;</comment>
                            <comment id="1010075" author="milkie" created="Wed, 19 Aug 2015 21:01:52 +0000"  >&lt;p&gt;I don&apos;t see why any oplog surgery is necessary &amp;#8211; their oplogs are still correct, just getting stale.  They crashed because they used a query on their sync source that didn&apos;t precisely overlap with what they already had in their oplog (is my theory).  If you attempt to restart them, what happens?&lt;/p&gt;</comment>
                            <comment id="1010068" author="dynamike" created="Wed, 19 Aug 2015 20:53:02 +0000"  >&lt;p&gt;Gotcha, but given that it happened in 2.6 means the code does get exercised. Is there any sort of oplog surgery I could do to the crashed 3.0 nodes rather than syncing from scratch?&lt;/p&gt;</comment>
                            <comment id="1010055" author="milkie" created="Wed, 19 Aug 2015 20:45:40 +0000"  >&lt;p&gt;Thanks for the logs!  I&apos;ll take a look.&lt;br/&gt;
I removed the force-sync-from-primary logic as part of a big refactor because that particular bandaid was a layer violation and it would have taken a big effort to reproduce the behavior it provided.  It&apos;s a condition that should be prohibited by the design of the logic, so I was comfortable with shutting down the server at that point so that a full diagnosis could take place.&lt;/p&gt;</comment>
                            <comment id="1010043" author="dynamike" created="Wed, 19 Aug 2015 20:32:46 +0000"  >&lt;p&gt;Log files are attached as hostnames. The state of the replicaset before the crash was &lt;/p&gt;

&lt;p&gt;db111 - PRIMARY running 2.6.7&lt;br/&gt;
db109 - SECONDARY running 2.6.7&lt;br/&gt;
db110 - SECONDARY running 2.6.7&lt;br/&gt;
db112 - SECONDARY running 2.6.7&lt;br/&gt;
md3-db1 - SECONDARY running 3.0.5&lt;br/&gt;
md3-db2 - SECONDARY running 3.0.5&lt;br/&gt;
md3-db3 - SECONDARY running 3.0.5&lt;/p&gt;

&lt;p&gt;I triggered an election to make md3-db1 PRIMARY and both md3-db2 and md3-db3 crashed with the &quot;back in time fassert&quot;. The other 2.6.7 SECONDARY nodes were fine.&lt;/p&gt;

&lt;p&gt;I noticed that after the election db111(2.6.7) also logged a back in time error, but the logic in 2.6 is to force sync from the PRIMARY. &lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Wed Aug 19 15:35:32.006 [rsSync] replication oplog stream went back in time. previous timestamp: 55d4a23f:d2 newest timestamp: 55d4a23f:cf. attempting to sync directly from primary.&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;The behavior change between 3.0 and 2.6 comes from this diff &amp;#8211;&lt;br/&gt;
&lt;a href=&quot;https://github.com/mongodb/mongo/commit/e86e08deff7293b5778fad27df9031c013595b12#diff-c64a61c81bff3e9fbae542a82463b76d&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://github.com/mongodb/mongo/commit/e86e08deff7293b5778fad27df9031c013595b12#diff-c64a61c81bff3e9fbae542a82463b76d&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Why was it decided to fassert rather than force sync from the primary?&lt;/p&gt;



</comment>
                            <comment id="1009897" author="dynamike" created="Wed, 19 Aug 2015 18:48:00 +0000"  >&lt;p&gt;sorry for the typo. Correct this issue is new in 3.0.5 .&lt;/p&gt;</comment>
                            <comment id="1009884" author="milkie" created="Wed, 19 Aug 2015 18:35:51 +0000"  >&lt;p&gt;I&apos;m sorry, did you mean to say you have experienced this issue only after upgrading to 3.0.5?  That is what the original description says.&lt;/p&gt;</comment>
                            <comment id="1009856" author="dynamike" created="Wed, 19 Aug 2015 18:11:07 +0000"  >&lt;p&gt;I&apos;ll work on getting the logs. I suspect the issue could be related to  &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-19375&quot; title=&quot;choosing syncsource should compare against last fetched optime rather than last applied&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-19375&quot;&gt;&lt;del&gt;SERVER-19375&lt;/del&gt;&lt;/a&gt;. We haven&apos;t had this issues since upgrading to 3.0.5 from 3.0.4&lt;/p&gt;</comment>
                            <comment id="1009850" author="milkie" created="Wed, 19 Aug 2015 18:01:00 +0000"  >&lt;p&gt;You&apos;re correct that there is one op that is being applied twice.  However, the sync source logic should prevent this from happening.&lt;br/&gt;
Do you have the full logs from the failing node and its sync source node for an instance of this issue?&lt;/p&gt;</comment>
                            <comment id="1009002" author="dynamike" created="Tue, 18 Aug 2015 23:50:14 +0000"  >&lt;p&gt;I put the crashed secondary in standalone mode took a look at the oplog by running &lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;db.oplog.rs.find( { }, { ts: 1, h: 1 } ).sort( {$natural : -1} )&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{ &quot;ts&quot; : Timestamp(1439937756, 70), &quot;h&quot; : NumberLong(&quot;1660986903740963784&quot;) }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{ &quot;ts&quot; : Timestamp(1439937756, 69), &quot;h&quot; : NumberLong(&quot;-5240046610690244083&quot;) }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{ &quot;ts&quot; : Timestamp(1439937756, 68), &quot;h&quot; : NumberLong(&quot;364598519254495583&quot;) }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{ &quot;ts&quot; : Timestamp(1439937756, 67), &quot;h&quot; : NumberLong(&quot;-1977037730287425552&quot;) }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;{ &quot;ts&quot; : Timestamp(1439937756, 66), &quot;h&quot; : NumberLong(&quot;1144686957426521030&quot;) }&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;&lt;/p&gt;

&lt;p&gt;It looks like the op is trying to be applied twice&lt;/p&gt;</comment>
                            <comment id="1008959" author="dynamike" created="Tue, 18 Aug 2015 23:12:03 +0000"  >&lt;p&gt;Tue Aug 18 22:42:38.336 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; New replica set config in use: { _id: &quot;blah&quot;, version: 83, members: [ { _id: 9, host: &quot;db109:27017&quot;, arbiterOnly: false, buildIndexes: true, hidden: false, priority: 2.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 10, host: &quot;db110:27017&quot;, arbiterOnly: false, buildIndexes: true, hidden: false, priority: 2.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 11, host: &quot;db111:27017&quot;, arbiterOnly: false, buildIndexes: true, hidden: false, priority: 2.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 12, host: &quot;db112:27017&quot;, arbiterOnly: false, buildIndexes: true, hidden: true, priority: 0.0, tags: {}, slaveDelay: 0, votes: 0 }, { _id: 13, host: &quot;arbiter3:30002&quot;, arbiterOnly: true, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 14, host: &quot;arbiter4:30002&quot;, arbiterOnly: true, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 15, host: &quot;db3:27017&quot;, arbiterOnly: false, buildIndexes: true, hidden: false, priority: 2.0, tags: {}, slaveDelay: 0, votes: 0 }, { _id: 16, host: &quot;db2:27017&quot;, arbiterOnly: false, buildIndexes: true, hidden: false, priority: 2.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 17, host: &quot;db1:27017&quot;, arbiterOnly: false, buildIndexes: true, hidden: false, priority: 3.0, tags: {}, slaveDelay: 0, votes: 1 } ], settings: { chainingAllowed: true, heartbeatTimeoutSecs: 10, getLastErrorModes: {}, getLastErrorDefaults: &lt;/p&gt;
{ w: 1, wtimeout: 0 }
&lt;p&gt; } }&lt;br/&gt;
Tue Aug 18 22:42:38.336 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; This node is db2:27017 in the config&lt;br/&gt;
Tue Aug 18 22:42:38.337 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Member db111:27017 is now in state SECONDARY&lt;br/&gt;
Tue Aug 18 22:42:38.337 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Standing for election&lt;br/&gt;
Tue Aug 18 22:42:38.337 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; not electing self, db109:27017 would veto with &apos;db2:27017 has lower priority than db1:27017&apos;&lt;br/&gt;
Tue Aug 18 22:42:38.337 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; not electing self, we are not freshest&lt;br/&gt;
Tue Aug 18 22:42:38.338 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Standing for election&lt;br/&gt;
Tue Aug 18 22:42:38.338 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Member db1:27017 is now in state PRIMARY&lt;br/&gt;
Tue Aug 18 22:42:38.338 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; not electing self, db109:27017 would veto with &apos;db2:27017 has lower priority than db1:27017&apos;&lt;br/&gt;
Tue Aug 18 22:42:40.670 F REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsSync&amp;#93;&lt;/span&gt; replication oplog stream went back in time. previous timestamp: 55d3b4dc:46 newest timestamp: 55d3b4dc:46. Op being applied: { ts: Timestamp 1439937756000|70, h: 1660986903740963784, v: 2, op: &quot;i&quot;, ns: &quot;...&quot;, o: &lt;/p&gt;
{ ..}
&lt;p&gt;Tue Aug 18 22:42:40.671 I -        &lt;span class=&quot;error&quot;&gt;&amp;#91;rsSync&amp;#93;&lt;/span&gt; Fatal Assertion 18905&lt;br/&gt;
Tue Aug 18 22:42:40.671 I -        &lt;span class=&quot;error&quot;&gt;&amp;#91;rsSync&amp;#93;&lt;/span&gt; &lt;/p&gt;

&lt;p&gt;***aborting after fassert() failure&lt;/p&gt;</comment>
                            <comment id="1008955" author="dynamike" created="Tue, 18 Aug 2015 23:07:22 +0000"  >&lt;p&gt;We&apos;ve hit this issue again. What could cause this to happen? &lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                                        </outwardlinks>
                                                                <inwardlinks description="is related to">
                                        <issuelink>
            <issuekey id="245273">SERVER-22030</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="86764" name="db111.gz" size="1970265" author="dynamike" created="Wed, 19 Aug 2015 20:32:46 +0000"/>
                            <attachment id="86763" name="md3-db1.gz" size="503757" author="dynamike" created="Wed, 19 Aug 2015 20:32:46 +0000"/>
                            <attachment id="86807" name="md3-db2-full.log.tar.gz" size="503757" author="dynamike" created="Thu, 20 Aug 2015 00:02:37 +0000"/>
                            <attachment id="86762" name="md3-db2.gz" size="43105" author="dynamike" created="Wed, 19 Aug 2015 20:32:46 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>25.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18555" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname># of Sprints</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>6.0</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Wed, 19 Aug 2015 03:02:57 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        8 years, 9 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>bernard.gorman@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            8 years, 9 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>milkie@mongodb.com</customfieldvalue>
            <customfieldvalue>ma6174</customfieldvalue>
            <customfieldvalue>dynamike</customfieldvalue>
            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>
            <customfieldvalue>tredman@fb.com</customfieldvalue>
            <customfieldvalue>udit</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrkx5z:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrdjw7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_10557" key="com.pyxis.greenhopper.jira:gh-sprint">
                        <customfieldname>Sprint</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue id="721">Repl C (11/20/15)</customfieldvalue>
    <customfieldvalue id="754">Repl D (12/11/15)</customfieldvalue>
    <customfieldvalue id="803">Repl E (01/08/16)</customfieldvalue>
    <customfieldvalue id="847">Repl F (01/29/16)</customfieldvalue>
    <customfieldvalue id="848">Repl 10 (02/19/16)</customfieldvalue>
    <customfieldvalue id="849">Repl 11 (03/11/16)</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hsfqlz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>