<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 04:15:28 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-27547] Adjust the system time . then MongoDB status maintian RECOVERING</title>
                <link>https://jira.mongodb.org/browse/SERVER-27547</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;grep -v &quot;[initandlisten]&amp;#124;[conn&quot;  log&lt;/p&gt;

&lt;p&gt;--------------------------------&lt;del&gt;node 1&lt;/del&gt;-------------------------------------&lt;br/&gt;
2017-03-14T14:41:57.607+0800 I STORAGE  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; shutdown: removing fs lock...&lt;br/&gt;
2017-03-14T14:41:57.608+0800 I CONTROL  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; dbexit:  rc: 0&lt;br/&gt;
2016-12-17T12:50:25.846+0800 I CONTROL  &lt;span class=&quot;error&quot;&gt;&amp;#91;main&amp;#93;&lt;/span&gt; ***** SERVER RESTARTED *****&lt;br/&gt;
2016-12-17T12:50:26.921+0800 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;HostnameCanonicalizationWorker&amp;#93;&lt;/span&gt; Starting hostname canonicalization worker&lt;br/&gt;
2016-12-17T12:50:26.965+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; New replica set config in use: { _id: &quot;fsp&quot;, version: 1, protocolVersion: 1, members: [ { _id: 0, host: &quot;172.28.1.140:27017&quot;, arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 1, host: &quot;172.28.1.141:27017&quot;, arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 2, host: &quot;172.28.1.142:27017&quot;, arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 } ], settings: { chainingAllowed: true, heartbeatIntervalMillis: 2000, heartbeatTimeoutSecs: 10, electionTimeoutMillis: 10000, getLastErrorModes: {}, getLastErrorDefaults: &lt;/p&gt;
{ w: 1, wtimeout: 0 }
&lt;p&gt;, replicaSetId: ObjectId(&apos;58c552b12d4be615ca28e5b1&apos;) } }&lt;br/&gt;
2016-12-17T12:50:26.965+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; This node is 172.28.1.140:27017 in the config&lt;br/&gt;
2016-12-17T12:50:26.965+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; transition to STARTUP2&lt;br/&gt;
2016-12-17T12:50:26.965+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Starting replication applier threads&lt;br/&gt;
2016-12-17T12:50:26.965+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.141:27017&lt;br/&gt;
2016-12-17T12:50:26.965+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; transition to RECOVERING&lt;br/&gt;
2016-12-17T12:50:26.965+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.142:27017&lt;br/&gt;
2016-12-17T12:50:26.966+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; transition to SECONDARY&lt;br/&gt;
2016-12-17T12:50:27.021+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.141:27017&lt;br/&gt;
2016-12-17T12:50:27.021+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.142:27017&lt;br/&gt;
2016-12-17T12:50:27.021+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Member 172.28.1.141:27017 is now in state SECONDARY&lt;br/&gt;
2016-12-17T12:50:27.022+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Member 172.28.1.142:27017 is now in state PRIMARY&lt;br/&gt;
2016-12-17T12:51:52.989+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; syncing from: 172.28.1.142:27017&lt;br/&gt;
2016-12-17T12:51:53.013+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-BGSync-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.142:27017&lt;br/&gt;
2016-12-17T12:51:53.013+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;SyncSourceFeedback&amp;#93;&lt;/span&gt; setting syncSourceFeedback to 172.28.1.142:27017&lt;br/&gt;
2016-12-17T12:51:53.045+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-BGSync-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.142:27017&lt;br/&gt;
2016-12-17T12:51:53.048+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; Starting rollback due to OplogStartMissing: our last op time fetched: (term: 11, timestamp: Mar 14 14:47:57:3c9d). source&apos;s GTE: (term: 12, timestamp: Mar 14 14:47:57:3c9d) hashes: (-1985836079535698320/4988593413792691282)&lt;br/&gt;
2016-12-17T12:51:53.050+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; beginning rollback&lt;br/&gt;
2016-12-17T12:51:53.050+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 0&lt;br/&gt;
2016-12-17T12:51:53.050+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; transition to ROLLBACK&lt;br/&gt;
2016-12-17T12:51:53.052+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 1&lt;br/&gt;
2016-12-17T12:51:53.079+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 2 FindCommonPoint&lt;br/&gt;
2016-12-17T12:51:53.080+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback our last optime:   Mar 14 14:47:57:3c9d&lt;br/&gt;
2016-12-17T12:51:53.080+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback their last optime: Mar 14 14:47:57:3d65&lt;br/&gt;
2016-12-17T12:51:53.080+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback diff in end of log times: 0 seconds&lt;br/&gt;
2016-12-17T12:51:53.343+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 3 fixup&lt;br/&gt;
2016-12-17T12:51:53.362+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 3.5&lt;br/&gt;
2016-12-17T12:51:53.363+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 4 n:2&lt;br/&gt;
2016-12-17T12:51:53.363+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; minvalid=(term: 12, timestamp: Mar 14 14:47:57:3d70)&lt;br/&gt;
2016-12-17T12:51:53.364+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 4.6&lt;br/&gt;
2016-12-17T12:51:53.364+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 4.7&lt;br/&gt;
2016-12-17T12:51:53.375+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 5 d:12 u:11&lt;br/&gt;
2016-12-17T12:51:53.376+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 6&lt;br/&gt;
2016-12-17T12:51:53.376+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback done&lt;br/&gt;
2016-12-17T12:51:53.376+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback finished&lt;br/&gt;
2016-12-17T12:51:53.376+0800 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;SyncSourceFeedback&amp;#93;&lt;/span&gt; Socket say send() errno:9 Bad file descriptor 172.28.1.142:27017&lt;br/&gt;
2016-12-17T12:51:53.376+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;SyncSourceFeedback&amp;#93;&lt;/span&gt; SyncSourceFeedback error sending update: socket exception &lt;span class=&quot;error&quot;&gt;&amp;#91;SEND_ERROR&amp;#93;&lt;/span&gt; for 172.28.1.142:27017&lt;br/&gt;
2016-12-17T12:51:53.376+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;SyncSourceFeedback&amp;#93;&lt;/span&gt; updateUpstream failed: Location9001: socket exception &lt;span class=&quot;error&quot;&gt;&amp;#91;SEND_ERROR&amp;#93;&lt;/span&gt; for 172.28.1.142:27017, will retry&lt;br/&gt;
2016-12-17T12:51:53.387+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; transition to RECOVERING&lt;br/&gt;
2016-12-17T12:51:53.388+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; syncing from: 172.28.1.141:27017&lt;br/&gt;
2016-12-17T12:51:53.416+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;SyncSourceFeedback&amp;#93;&lt;/span&gt; setting syncSourceFeedback to 172.28.1.141:27017&lt;br/&gt;
2016-12-17T12:51:53.416+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-BGSync-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.141:27017&lt;br/&gt;
2016-12-17T12:51:53.446+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-BGSync-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.141:27017&lt;br/&gt;
2016-12-17T12:51:57.039+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; dropping unhealthy pooled connection to 172.28.1.141:27017&lt;br/&gt;
2016-12-17T12:51:57.039+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; after drop, pool was empty, going to spawn some connections&lt;br/&gt;
2016-12-17T12:51:57.040+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.141:27017&lt;br/&gt;
2016-12-17T12:51:57.076+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.141:27017&lt;br/&gt;
2016-12-17T12:51:57.077+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Member 172.28.1.141:27017 is now in state RECOVERING&lt;br/&gt;
2016-12-17T15:13:45.705+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-BGSync-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.141:27017&lt;/p&gt;

&lt;p&gt;--------------------------------&lt;del&gt;node 2&lt;/del&gt;-------------------------------------&lt;br/&gt;
2017-03-14T14:41:50.897+0800 I CONTROL  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; now exiting&lt;br/&gt;
2017-03-14T14:41:50.897+0800 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; shutdown: going to close listening sockets...&lt;br/&gt;
2017-03-14T14:41:50.897+0800 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; closing listening socket: 7&lt;br/&gt;
2017-03-14T14:41:50.897+0800 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; shutdown: going to flush diaglog...&lt;br/&gt;
2017-03-14T14:41:50.897+0800 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; shutdown: going to close sockets...&lt;br/&gt;
2017-03-14T14:41:50.897+0800 I STORAGE  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; WiredTigerKVEngine shutting down&lt;br/&gt;
2017-03-14T14:41:50.913+0800 I STORAGE  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; shutdown: removing fs lock...&lt;br/&gt;
2017-03-14T14:41:50.913+0800 I CONTROL  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; dbexit:  rc: 0&lt;br/&gt;
2016-12-17T12:50:12.767+0800 I CONTROL  &lt;span class=&quot;error&quot;&gt;&amp;#91;main&amp;#93;&lt;/span&gt; ***** SERVER RESTARTED *****&lt;br/&gt;
2016-12-17T12:50:13.573+0800 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;HostnameCanonicalizationWorker&amp;#93;&lt;/span&gt; Starting hostname canonicalization worker&lt;br/&gt;
2016-12-17T12:50:16.580+0800 W NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Failed to connect to 172.28.1.140:27017, reason: errno:113 No route to host&lt;br/&gt;
2016-12-17T12:50:16.599+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; New replica set config in use: { _id: &quot;fsp&quot;, version: 1, protocolVersion: 1, members: [ { _id: 0, host: &quot;172.28.1.140:27017&quot;, arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 1, host: &quot;172.28.1.141:27017&quot;, arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 2, host: &quot;172.28.1.142:27017&quot;, arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 } ], settings: { chainingAllowed: true, heartbeatIntervalMillis: 2000, heartbeatTimeoutSecs: 10, electionTimeoutMillis: 10000, getLastErrorModes: {}, getLastErrorDefaults: &lt;/p&gt;
{ w: 1, wtimeout: 0 }
&lt;p&gt;, replicaSetId: ObjectId(&apos;58c552b12d4be615ca28e5b1&apos;) } }&lt;br/&gt;
2016-12-17T12:50:16.599+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; This node is 172.28.1.141:27017 in the config&lt;br/&gt;
2016-12-17T12:50:16.599+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; transition to STARTUP2&lt;br/&gt;
2016-12-17T12:50:16.599+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Starting replication applier threads&lt;br/&gt;
2016-12-17T12:50:16.600+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:16.600+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; transition to RECOVERING&lt;br/&gt;
2016-12-17T12:50:16.600+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.142:27017&lt;br/&gt;
2016-12-17T12:50:16.601+0800 I COMMAND  &lt;span class=&quot;error&quot;&gt;&amp;#91;ftdc&amp;#93;&lt;/span&gt; serverStatus was very slow: &lt;/p&gt;
{ after basic: 0, after asserts: 0, after connections: 0, after extra_info: 0, after globalLock: 0, after locks: 0, after network: 0, after opcounters: 0, after opcountersRepl: 0, after repl: 2580, after storageEngine: 2580, after tcmalloc: 2580, after wiredTiger: 2590, at end: 2590 }
&lt;p&gt;2016-12-17T12:50:16.602+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; transition to SECONDARY&lt;br/&gt;
2016-12-17T12:50:16.994+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.142:27017&lt;br/&gt;
2016-12-17T12:50:16.998+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Member 172.28.1.142:27017 is now in state SECONDARY&lt;br/&gt;
2016-12-17T12:50:19.586+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Failed to connect to 172.28.1.140:27017 - HostUnreachable: No route to host&lt;br/&gt;
2016-12-17T12:50:19.587+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Error in heartbeat request to 172.28.1.140:27017; HostUnreachable: No route to host&lt;br/&gt;
2016-12-17T12:50:19.587+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:22.592+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Failed to connect to 172.28.1.140:27017 - HostUnreachable: No route to host&lt;br/&gt;
2016-12-17T12:50:22.593+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Error in heartbeat request to 172.28.1.140:27017; HostUnreachable: No route to host&lt;br/&gt;
2016-12-17T12:50:22.593+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:25.599+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Failed to connect to 172.28.1.140:27017 - HostUnreachable: No route to host&lt;br/&gt;
2016-12-17T12:50:25.599+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Error in heartbeat request to 172.28.1.140:27017; HostUnreachable: No route to host&lt;br/&gt;
2016-12-17T12:50:30.599+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:30.631+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:30.632+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Member 172.28.1.140:27017 is now in state SECONDARY&lt;br/&gt;
2016-12-17T12:50:31.614+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; syncing from: 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:31.648+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-BGSync-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:31.648+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;SyncSourceFeedback&amp;#93;&lt;/span&gt; setting syncSourceFeedback to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:31.677+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-BGSync-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:32.001+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Member 172.28.1.142:27017 is now in state PRIMARY&lt;br/&gt;
2016-12-17T12:50:36.684+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; could not find member to sync from&lt;br/&gt;
2016-12-17T12:51:52.704+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; syncing from: 172.28.1.142:27017&lt;br/&gt;
2016-12-17T12:51:52.729+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-BGSync-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.142:27017&lt;br/&gt;
2016-12-17T12:51:52.730+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;SyncSourceFeedback&amp;#93;&lt;/span&gt; setting syncSourceFeedback to 172.28.1.142:27017&lt;br/&gt;
2016-12-17T12:51:52.752+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-BGSync-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.142:27017&lt;br/&gt;
2016-12-17T12:51:52.754+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; Starting rollback due to OplogStartMissing: our last op time fetched: (term: 11, timestamp: Mar 14 14:47:57:3c9d). source&apos;s GTE: (term: 12, timestamp: Mar 14 14:47:57:3c9d) hashes: (-1985836079535698320/4988593413792691282)&lt;br/&gt;
2016-12-17T12:51:52.754+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; beginning rollback&lt;br/&gt;
2016-12-17T12:51:52.754+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 0&lt;br/&gt;
2016-12-17T12:51:52.755+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; transition to ROLLBACK&lt;br/&gt;
2016-12-17T12:51:52.757+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 1&lt;br/&gt;
2016-12-17T12:51:52.784+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 2 FindCommonPoint&lt;br/&gt;
2016-12-17T12:51:52.785+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback our last optime:   Mar 14 14:47:57:3c9d&lt;br/&gt;
2016-12-17T12:51:52.785+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback their last optime: Mar 14 14:47:57:3d5e&lt;br/&gt;
2016-12-17T12:51:52.785+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback diff in end of log times: 0 seconds&lt;br/&gt;
2016-12-17T12:51:53.111+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 3 fixup&lt;br/&gt;
2016-12-17T12:51:53.124+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 3.5&lt;br/&gt;
2016-12-17T12:51:53.125+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 4 n:2&lt;br/&gt;
2016-12-17T12:51:53.125+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; minvalid=(term: 12, timestamp: Mar 14 14:47:57:3d68)&lt;br/&gt;
2016-12-17T12:51:53.125+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 4.6&lt;br/&gt;
2016-12-17T12:51:53.125+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 4.7&lt;br/&gt;
2016-12-17T12:51:53.129+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 5 d:12 u:11&lt;br/&gt;
2016-12-17T12:51:53.129+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback 6&lt;br/&gt;
2016-12-17T12:51:53.129+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback done&lt;br/&gt;
2016-12-17T12:51:53.129+0800 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;SyncSourceFeedback&amp;#93;&lt;/span&gt; Socket say send() errno:9 Bad file descriptor 172.28.1.142:27017&lt;br/&gt;
2016-12-17T12:51:53.129+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; rollback finished&lt;br/&gt;
2016-12-17T12:51:53.129+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;SyncSourceFeedback&amp;#93;&lt;/span&gt; SyncSourceFeedback error sending update: socket exception &lt;span class=&quot;error&quot;&gt;&amp;#91;SEND_ERROR&amp;#93;&lt;/span&gt; for 172.28.1.142:27017&lt;br/&gt;
2016-12-17T12:51:53.130+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;SyncSourceFeedback&amp;#93;&lt;/span&gt; updateUpstream failed: Location9001: socket exception &lt;span class=&quot;error&quot;&gt;&amp;#91;SEND_ERROR&amp;#93;&lt;/span&gt; for 172.28.1.142:27017, will retry&lt;br/&gt;
2016-12-17T12:51:53.139+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; transition to RECOVERING&lt;br/&gt;
2016-12-17T12:51:53.139+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; syncing from: 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:51:53.164+0800 E REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; sync producer problem: 13436 cannot read from oplog collection while in rollback&lt;br/&gt;
2016-12-17T12:51:53.265+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; syncing from: 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:51:53.286+0800 E REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; sync producer problem: 13436 cannot read from oplog collection while in rollback&lt;br/&gt;
2016-12-17T12:51:53.386+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; syncing from: 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:51:53.414+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;SyncSourceFeedback&amp;#93;&lt;/span&gt; setting syncSourceFeedback to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:51:53.414+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; dropping unhealthy pooled connection to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:51:53.414+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; after drop, pool was empty, going to spawn some connections&lt;br/&gt;
2016-12-17T12:51:53.414+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-BGSync-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:51:53.448+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-BGSync-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:51:56.702+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; dropping unhealthy pooled connection to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:51:56.702+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; after drop, pool was empty, going to spawn some connections&lt;br/&gt;
2016-12-17T12:51:56.703+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:51:56.723+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:51:56.723+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Member 172.28.1.140:27017 is now in state RECOVERING&lt;br/&gt;
2016-12-17T13:38:09.036+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-BGSync-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T13:38:09.059+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-BGSync-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T13:42:19.087+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-BGSync-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T13:42:19.108+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-BGSync-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T16:52:06.754+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-BGSync-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T16:52:06.781+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-BGSync-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T19:38:02.677+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;SyncSourceFeedback&amp;#93;&lt;/span&gt; SyncSourceFeedback error sending update: network error while attempting to run command &apos;replSetUpdatePosition&apos; on host &apos;172.28.1.140:27017&apos; &lt;br/&gt;
2016-12-17T19:38:02.677+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;SyncSourceFeedback&amp;#93;&lt;/span&gt; updateUpstream failed: HostUnreachable: network error while attempting to run command &apos;replSetUpdatePosition&apos; on host &apos;172.28.1.140:27017&apos; , will retry&lt;br/&gt;
2016-12-17T19:38:02.919+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Error in heartbeat request to 172.28.1.140:27017; HostUnreachable: End of file&lt;br/&gt;
2016-12-17T19:38:02.919+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;/p&gt;

&lt;p&gt;--------------------------------&lt;del&gt;node 3&lt;/del&gt;-------------------------------------&lt;br/&gt;
2017-03-14T14:41:50.894+0800 W EXECUTOR &lt;span class=&quot;error&quot;&gt;&amp;#91;rsBackgroundSync&amp;#93;&lt;/span&gt; killCursors command task failed: CallbackCanceled: Callback canceled&lt;br/&gt;
2017-03-14T14:41:50.895+0800 I CONTROL  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; now exiting&lt;br/&gt;
2017-03-14T14:41:50.895+0800 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; shutdown: going to close listening sockets...&lt;br/&gt;
2017-03-14T14:41:50.895+0800 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; closing listening socket: 7&lt;br/&gt;
2017-03-14T14:41:50.895+0800 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; shutdown: going to flush diaglog...&lt;br/&gt;
2017-03-14T14:41:50.895+0800 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; shutdown: going to close sockets...&lt;br/&gt;
2017-03-14T14:41:50.895+0800 I STORAGE  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; WiredTigerKVEngine shutting down&lt;br/&gt;
2017-03-14T14:41:51.034+0800 I STORAGE  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; shutdown: removing fs lock...&lt;br/&gt;
2017-03-14T14:41:51.034+0800 I CONTROL  &lt;span class=&quot;error&quot;&gt;&amp;#91;signalProcessingThread&amp;#93;&lt;/span&gt; dbexit:  rc: 0&lt;br/&gt;
2016-12-17T12:50:13.156+0800 I CONTROL  &lt;span class=&quot;error&quot;&gt;&amp;#91;main&amp;#93;&lt;/span&gt; ***** SERVER RESTARTED *****&lt;br/&gt;
2016-12-17T12:50:13.945+0800 I NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;HostnameCanonicalizationWorker&amp;#93;&lt;/span&gt; Starting hostname canonicalization worker&lt;br/&gt;
2016-12-17T12:50:16.951+0800 W NETWORK  &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Failed to connect to 172.28.1.140:27017, reason: errno:113 No route to host&lt;br/&gt;
2016-12-17T12:50:16.974+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; New replica set config in use: { _id: &quot;fsp&quot;, version: 1, protocolVersion: 1, members: [ { _id: 0, host: &quot;172.28.1.140:27017&quot;, arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 1, host: &quot;172.28.1.141:27017&quot;, arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 }, { _id: 2, host: &quot;172.28.1.142:27017&quot;, arbiterOnly: false, buildIndexes: true, hidden: false, priority: 1.0, tags: {}, slaveDelay: 0, votes: 1 } ], settings: { chainingAllowed: true, heartbeatIntervalMillis: 2000, heartbeatTimeoutSecs: 10, electionTimeoutMillis: 10000, getLastErrorModes: {}, getLastErrorDefaults: &lt;/p&gt;
{ w: 1, wtimeout: 0 }
&lt;p&gt;, replicaSetId: ObjectId(&apos;58c552b12d4be615ca28e5b1&apos;) } }&lt;br/&gt;
2016-12-17T12:50:16.974+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; This node is 172.28.1.142:27017 in the config&lt;br/&gt;
2016-12-17T12:50:16.974+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; transition to STARTUP2&lt;br/&gt;
2016-12-17T12:50:16.974+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Starting replication applier threads&lt;br/&gt;
2016-12-17T12:50:16.974+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:16.975+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.141:27017&lt;br/&gt;
2016-12-17T12:50:16.975+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; transition to RECOVERING&lt;br/&gt;
2016-12-17T12:50:16.976+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; transition to SECONDARY&lt;br/&gt;
2016-12-17T12:50:16.977+0800 I COMMAND  &lt;span class=&quot;error&quot;&gt;&amp;#91;ftdc&amp;#93;&lt;/span&gt; serverStatus was very slow: &lt;/p&gt;
{ after basic: 0, after asserts: 0, after connections: 0, after extra_info: 0, after globalLock: 0, after locks: 0, after network: 0, after opcounters: 0, after opcountersRepl: 0, after repl: 2958, after storageEngine: 2958, after tcmalloc: 2958, after wiredTiger: 2958, at end: 2958 }
&lt;p&gt;2016-12-17T12:50:17.003+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.141:27017&lt;br/&gt;
2016-12-17T12:50:17.003+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Member 172.28.1.141:27017 is now in state SECONDARY&lt;br/&gt;
2016-12-17T12:50:19.957+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Failed to connect to 172.28.1.140:27017 - HostUnreachable: No route to host&lt;br/&gt;
2016-12-17T12:50:19.957+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Error in heartbeat request to 172.28.1.140:27017; HostUnreachable: No route to host&lt;br/&gt;
2016-12-17T12:50:19.957+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:22.963+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Failed to connect to 172.28.1.140:27017 - HostUnreachable: No route to host&lt;br/&gt;
2016-12-17T12:50:22.963+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Error in heartbeat request to 172.28.1.140:27017; HostUnreachable: No route to host&lt;br/&gt;
2016-12-17T12:50:22.963+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:25.969+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Failed to connect to 172.28.1.140:27017 - HostUnreachable: No route to host&lt;br/&gt;
2016-12-17T12:50:25.970+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Error in heartbeat request to 172.28.1.140:27017; HostUnreachable: No route to host&lt;br/&gt;
2016-12-17T12:50:27.019+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Starting an election, since we&apos;ve seen no PRIMARY in the past 10000ms&lt;br/&gt;
2016-12-17T12:50:27.019+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; conducting a dry run election to see if we could be elected&lt;br/&gt;
2016-12-17T12:50:27.019+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:27.020+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; dry election run succeeded, running for election&lt;br/&gt;
2016-12-17T12:50:27.021+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:27.021+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; election succeeded, assuming primary role in term 12&lt;br/&gt;
2016-12-17T12:50:27.021+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; transition to PRIMARY&lt;br/&gt;
2016-12-17T12:50:27.021+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:27.070+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:27.070+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:27.071+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:50:27.075+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Member 172.28.1.140:27017 is now in state SECONDARY&lt;br/&gt;
2016-12-17T12:50:27.980+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;rsSync&amp;#93;&lt;/span&gt; transition to primary complete; database writes are now permitted&lt;br/&gt;
2016-12-17T12:51:53.053+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; dropping unhealthy pooled connection to 172.28.1.141:27017&lt;br/&gt;
2016-12-17T12:51:53.053+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; after drop, pool was empty, going to spawn some connections&lt;br/&gt;
2016-12-17T12:51:53.053+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.141:27017&lt;br/&gt;
2016-12-17T12:51:53.081+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.141:27017&lt;br/&gt;
2016-12-17T12:51:53.082+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Member 172.28.1.141:27017 is now in state ROLLBACK&lt;br/&gt;
2016-12-17T12:51:53.114+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; dropping unhealthy pooled connection to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:51:53.114+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; after drop, pool was empty, going to spawn some connections&lt;br/&gt;
2016-12-17T12:51:53.114+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:51:53.138+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Successfully connected to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T12:51:53.142+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Member 172.28.1.140:27017 is now in state ROLLBACK&lt;br/&gt;
2016-12-17T12:51:55.083+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Member 172.28.1.141:27017 is now in state RECOVERING&lt;br/&gt;
2016-12-17T12:51:55.143+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Member 172.28.1.140:27017 is now in state RECOVERING&lt;br/&gt;
2016-12-17T19:38:02.677+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Error in heartbeat request to 172.28.1.140:27017; HostUnreachable: End of file&lt;br/&gt;
2016-12-17T19:38:02.678+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Connecting to 172.28.1.140:27017&lt;br/&gt;
2016-12-17T19:38:02.679+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; Failed to connect to 172.28.1.140:27017 - HostUnreachable: End of file&lt;br/&gt;
2016-12-17T19:38:02.679+0800 I ASIO     &lt;span class=&quot;error&quot;&gt;&amp;#91;NetworkInterfaceASIO-Replication-0&amp;#93;&lt;/span&gt; failed to close stream: Transport endpoint is not connected&lt;br/&gt;
2016-12-17T19:38:02.679+0800 I REPL     &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicationExecutor&amp;#93;&lt;/span&gt; Error in heartbeat request to 172.28.1.140:27017; HostUnreachable: End of file&lt;/p&gt;</description>
                <environment></environment>
        <key id="341695">SERVER-27547</key>
            <summary>Adjust the system time . then MongoDB status maintian RECOVERING</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="3">Duplicate</resolution>
                                        <assignee username="spencer@mongodb.com">Spencer Brody</assignee>
                                    <reporter username="eveninger">eveninger</reporter>
                        <labels>
                    </labels>
                <created>Fri, 30 Dec 2016 06:39:05 +0000</created>
                <updated>Mon, 30 Jan 2017 20:36:34 +0000</updated>
                            <resolved>Mon, 30 Jan 2017 20:36:34 +0000</resolved>
                                    <version>3.2.10</version>
                                                    <component>Replication</component>
                                        <votes>0</votes>
                                    <watches>7</watches>
                                                                                                                <comments>
                            <comment id="1488446" author="thomas.schubert" created="Mon, 30 Jan 2017 20:36:00 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=eveninger&quot; class=&quot;user-hover&quot; rel=&quot;eveninger&quot;&gt;eveninger&lt;/a&gt;, &lt;/p&gt;

&lt;p&gt;We haven&#8217;t heard back from you for some time, so I&#8217;m going to mark this ticket as resolved. If you would like us to take a look at the logs, please attach them and we will reopen the ticket.&lt;/p&gt;

&lt;p&gt;Regards,&lt;br/&gt;
Thomas&lt;/p&gt;</comment>
                            <comment id="1470670" author="spencer" created="Thu, 5 Jan 2017 23:56:19 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=eveninger&quot; class=&quot;user-hover&quot; rel=&quot;eveninger&quot;&gt;eveninger&lt;/a&gt;,&lt;br/&gt;
There are 3 components to the OpTime (the unique id of each oplog entry which must be increasing to preserve proper oplog ordering).  The first is the term, which is incremented in every election attempt.  There can only be 1 primary in a given term, so oplog entries in higher terms are guaranteed to have come from primaries that were elected later.  After the term comes the Timestamp, which in turn has 2 components: &apos;ts&apos;, and &apos;i&apos;.  When an oplog entry is generated, the &apos;ts&apos; field is initialized to either the current time in seconds from reading the system clock, or the &apos;ts&apos; value of the last timestamp assigned to the last oplog entry in that nodes oplog, whichever is higher.  This should prevent setting the clock backwards from generating OpTimes that go backwards.  Finally, the &apos;i&apos; field starts at 0 but if the &apos;ts&apos; field of the last oplog entry is the same as the &apos;ts&apos; field of the entry being generated (either because multiple writes happened in the same second or because the time was set backwards at some point and system time hasn&apos;t caught up to where the clock was when it was ahead), then &apos;i&apos; becomes the &apos;i&apos; of the previous oplog entry + 1.&lt;/p&gt;

&lt;p&gt;As to why the rollback happened, that is hard to say without seeing fuller logs including the time period leading up to where the nodes were shut down and the clock was adjusted.  If you attach full logs covering the same time range you posted above as well as 10 minutes leading up to where the nodes were restarted, I would be happy to take a look and see if I can understand why the rollback happened in the first place.&lt;/p&gt;</comment>
                            <comment id="1467294" author="eveninger" created="Sat, 31 Dec 2016 02:01:51 +0000"  >&lt;p&gt;think you Spencer. I have some doubts.&lt;br/&gt;
Who triggered the rollback? I mean if nobody adjust the clock, why node 1 and 2 go into rollback at the same time?&lt;/p&gt;

&lt;p&gt;in my opinion&#65292;oplog timestamp like this :&lt;br/&gt;
node 1: 1-2-3-(adjustment)-5&lt;br/&gt;
primary: 1-2-3-(adjustment)-4-6&lt;br/&gt;
so node 1 must rollback to : 1-2-3, then sync 4-6 from the primary for a consistent state.&lt;/p&gt;

&lt;p&gt;how the oplog manager continuity? timestamp or position&#65288;AUTO_INCREMENT sequence&#65289;?&lt;br/&gt;
if use timestamp, when I adjust the clock from 2017-03-14 to 2016-12-17, and insert some new entries, oplog has become discontinuous.&lt;/p&gt;</comment>
                            <comment id="1467148" author="spencer" created="Fri, 30 Dec 2016 18:32:56 +0000"  >&lt;p&gt;Hi &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=eveninger&quot; class=&quot;user-hover&quot; rel=&quot;eveninger&quot;&gt;eveninger&lt;/a&gt;,&lt;br/&gt;
I&apos;ve read through the logs you posted, and from the logs I don&apos;t think the issue you&apos;re hitting is related to the clock adjustment, though there is definitely a real bug in play here.  From the logs you can see that both node 1 and 2 go into rollback, rolling back against the primary.  The problem comes when the nodes transition from ROLLBACK into RECOVERING, where they are looking for oplog entries to apply to bring them back into a consistent state.  The problem is that node 1 winds up trying to sync ops from node 2, at the same time as node 2 is trying to sync ops from node 1, thus neither is ever able to catch up.  I believe this was due to a narrow race condition resulting from the combination of &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-25145&quot; title=&quot;During rollback (or w/minvalid invalid) select sync source based on end OpTime&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-25145&quot;&gt;&lt;del&gt;SERVER-25145&lt;/del&gt;&lt;/a&gt; and &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-27403&quot; title=&quot;Consider term and rbid when validating the proposed sync source&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-27403&quot;&gt;&lt;del&gt;SERVER-27403&lt;/del&gt;&lt;/a&gt;.  I&apos;m sorry that you encountered this bug!  &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-25145&quot; title=&quot;During rollback (or w/minvalid invalid) select sync source based on end OpTime&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-25145&quot;&gt;&lt;del&gt;SERVER-25145&lt;/del&gt;&lt;/a&gt; has been fixed in 3.2.11, and would be sufficient to prevent this from occurring, so I recommend you upgrade to avoid hitting this in the future.  Additional work on &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-27403&quot; title=&quot;Consider term and rbid when validating the proposed sync source&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-27403&quot;&gt;&lt;del&gt;SERVER-27403&lt;/del&gt;&lt;/a&gt; is ongoing and we hope to have ready in time for 3.2.12, which will provide additional protections in the area of sync source selection during rollbacks.&lt;/p&gt;

&lt;p&gt;Cheers,&lt;br/&gt;
-Spencer&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10010">
                    <name>Duplicate</name>
                                            <outwardlinks description="duplicates">
                                        <issuelink>
            <issuekey id="302251">SERVER-25145</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="338476">SERVER-27403</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                            <attachment id="146843" name="screenshot-1.png" size="734" author="eveninger" created="Sat, 31 Dec 2016 01:34:39 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>4.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 30 Dec 2016 14:57:23 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        7 years, 2 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>kelsey.schubert@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            7 years, 2 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>eveninger</customfieldvalue>
            <customfieldvalue>kelsey.schubert@mongodb.com</customfieldvalue>
            <customfieldvalue>spencer@mongodb.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|ht0aw7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hst9r3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9223372036854775807</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10750" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Steps To Reproduce</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>&lt;p&gt;0&#12289;Replica set with 3 nodes&lt;br/&gt;
1&#12289;shutdown all 3 nodes&lt;br/&gt;
2&#12289;adjust the system time from 2017-03-14 to 2016-12-17 &#65288;date -s&#65289; &lt;br/&gt;
3&#12289;start all 3 nodes&lt;br/&gt;
4&#12289;one PRIMARY, other two nodes maintian RECOVERING&lt;/p&gt;

</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hs4kdz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>