<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:02:09 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-3125] replset synchronisation performance problem (storing files)</title>
                <link>https://jira.mongodb.org/browse/SERVER-3125</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;My java app is storing messages and files in mongo.&lt;/p&gt;

&lt;p&gt;Secondary nodes couldn&apos;t  receive data enough fast to replicate changes.&lt;/p&gt;


&lt;p&gt;Set name:	testreplset&lt;br/&gt;
Majority up:	yes&lt;br/&gt;
Member	id	Up	cctime	Last heartbeat	Votes	Priority	State	Messages	optime	skew&lt;br/&gt;
172.17.0.251:27017	0	1	74 hrs	1 sec ago	1	1	RECOVERING	error RS102 too stale to catch up	4dd6539b:81	&lt;br/&gt;
172.17.0.252:27017 (me)	1	1	74 hrs		1	1	RECOVERING	error RS102 too stale to catch up	4dd6546d:7c	&lt;br/&gt;
172.17.0.253:27017	2	1	74 hrs	1 sec ago	1	1	PRIMARY		4dd65853:8	     &lt;/p&gt;


&lt;p&gt;     12:28:59 &lt;span class=&quot;error&quot;&gt;&amp;#91;startReplSets&amp;#93;&lt;/span&gt; replSet STARTUP2&lt;br/&gt;
           12:28:59 &lt;span class=&quot;error&quot;&gt;&amp;#91;rs Manager&amp;#93;&lt;/span&gt; replSet can&apos;t see a majority, will not try to elect self&lt;br/&gt;
           12:28:59 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet SECONDARY&lt;br/&gt;
           12:28:59 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet info 172.17.0.253:27017 is up&lt;br/&gt;
           12:28:59 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet member 172.17.0.253:27017 SECONDARY&lt;br/&gt;
           12:29:00 &lt;span class=&quot;error&quot;&gt;&amp;#91;rs Manager&amp;#93;&lt;/span&gt; replSet info electSelf 1&lt;br/&gt;
           12:29:00 &lt;span class=&quot;error&quot;&gt;&amp;#91;rs Manager&amp;#93;&lt;/span&gt; replSet couldn&apos;t elect self, only received -9999 votes&lt;br/&gt;
           12:29:01 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2&amp;#93;&lt;/span&gt; replSet info voting yea for 2&lt;br/&gt;
           12:29:01 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet member 172.17.0.253:27017 PRIMARY&lt;br/&gt;
           12:29:01 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet info 172.17.0.251:27017 is up&lt;br/&gt;
           12:29:01 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet member 172.17.0.251:27017 SECONDARY&lt;br/&gt;
           12:29:10 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet syncThread: 10278 dbclient error communicating with server: 172.17.0.253:27017&lt;br/&gt;
           12:29:11 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet info 172.17.0.253:27017 is down (or slow to respond): DBClientBase::findOne: transport error: 172.17.0.253:27017 query: { replSetHeartbeat: &quot;testreplset&quot;, v: 1, pv: 1, checkEmpty: false, from: &quot;           12:29:12 &lt;span class=&quot;error&quot;&gt;&amp;#91;rs Manager&amp;#93;&lt;/span&gt; replSet not trying to elect self as responded yea to someone else recently&lt;br/&gt;
           12:29:18 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn6&amp;#93;&lt;/span&gt; replSet info voting yea for 2&lt;br/&gt;
           12:29:18 &lt;span class=&quot;error&quot;&gt;&amp;#91;rs Manager&amp;#93;&lt;/span&gt; replSet not trying to elect self as responded yea to someone else recently&lt;br/&gt;
           12:29:19 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet info 172.17.0.253:27017 is up&lt;br/&gt;
Fri May 20 13:45:44 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet member 172.17.0.251:27017 RECOVERING&lt;br/&gt;
           13:48:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up, at least from 172.17.0.253:27017&lt;br/&gt;
           13:48:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet our last optime : May 20 13:45:49 4dd6546d:7c&lt;br/&gt;
           13:48:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet oldest at 172.17.0.253:27017 : May 20 13:45:52 4dd65470:19&lt;br/&gt;
           13:48:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet See &lt;a href=&quot;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&lt;/a&gt;&lt;br/&gt;
           13:48:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up&lt;br/&gt;
           13:48:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet RECOVERING&lt;br/&gt;
           13:49:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet member 172.17.0.251:27017 SECONDARY&lt;br/&gt;
           13:49:48 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet member 172.17.0.251:27017 RECOVERING&lt;br/&gt;
           13:50:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up, at least from 172.17.0.253:27017&lt;br/&gt;
           13:50:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet our last optime : May 20 13:45:49 4dd6546d:7c&lt;br/&gt;
           13:50:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet oldest at 172.17.0.253:27017 : May 20 13:47:42 4dd654de:32&lt;br/&gt;
           13:50:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet See &lt;a href=&quot;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&lt;/a&gt;&lt;br/&gt;
           13:50:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up&lt;br/&gt;
           13:52:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet SECONDARY&lt;br/&gt;
           13:52:56 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up, at least from 172.17.0.253:27017&lt;br/&gt;
           13:52:56 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet our last optime : May 20 13:45:49 4dd6546d:7c&lt;br/&gt;
           13:52:56 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet oldest at 172.17.0.253:27017 : May 20 13:49:51 4dd6555f:162&lt;br/&gt;
           13:52:56 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet See &lt;a href=&quot;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&lt;/a&gt;&lt;br/&gt;
           13:52:56 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up&lt;br/&gt;
           13:52:56 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet RECOVERING&lt;br/&gt;
           13:54:56 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up, at least from 172.17.0.253:27017&lt;br/&gt;
           13:54:56 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet our last optime : May 20 13:45:49 4dd6546d:7c&lt;br/&gt;
           13:54:56 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet oldest at 172.17.0.253:27017 : May 20 13:51:43 4dd655cf:47&lt;br/&gt;
           13:54:56 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet See &lt;a href=&quot;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&lt;/a&gt;&lt;br/&gt;
           13:54:56 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up&lt;/p&gt;</description>
                <environment>linux 2.6.38 kernel, 4GB ram, 2GB swap for each node, 3x nodex, 4th computer with application in java. connection between computers - 100mbps LAN, default oplog size (no oplogSize directive in configuration file)</environment>
        <key id="17307">SERVER-3125</key>
            <summary>replset synchronisation performance problem (storing files)</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="kristina">Kristina Chodorow</assignee>
                                    <reporter username="msz">MartinS</reporter>
                        <labels>
                    </labels>
                <created>Fri, 20 May 2011 12:13:49 +0000</created>
                <updated>Wed, 29 Feb 2012 03:54:06 +0000</updated>
                            <resolved>Wed, 16 Nov 2011 18:30:23 +0000</resolved>
                                    <version>1.8.1</version>
                                                    <component>Replication</component>
                                        <votes>0</votes>
                                    <watches>3</watches>
                                                                                                                <comments>
                            <comment id="67201" author="kristina" created="Wed, 16 Nov 2011 18:30:23 +0000"  >&lt;p&gt;Okay, glad that&apos;s working for you.&lt;/p&gt;</comment>
                            <comment id="63474" author="msz" created="Sat, 29 Oct 2011 04:34:43 +0000"  >&lt;p&gt;better hardware isn&apos;t solution.&lt;/p&gt;

&lt;p&gt;with 1Gbps LAN (I&apos;ve recently&lt;/p&gt;

&lt;p&gt;The best way to solve this is to set 2 limit:&lt;br/&gt;
oplogSize=2GB - limit oplog in normal state&lt;br/&gt;
and&lt;br/&gt;
maxOplogSize=30GB - this is used when writing data is faster than replication.&lt;/p&gt;
</comment>
                            <comment id="63419" author="kristina" created="Fri, 28 Oct 2011 20:27:06 +0000"  >&lt;p&gt;(Yay, you&apos;re still around!)&lt;/p&gt;

&lt;p&gt;In the other ticket (rollback, &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-3126&quot; title=&quot;replset - common point problem&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-3126&quot;&gt;&lt;del&gt;SERVER-3126&lt;/del&gt;&lt;/a&gt;), it says your primary was 998 seconds ahead (@ 4MB/sec), meaning it was ~4GB ahead. It sounds like you&apos;re just writing faster than MongoDB can replicate.  You could:&lt;/p&gt;

&lt;ol&gt;
	&lt;li&gt;throttle writes using w=&quot;majority&quot;&lt;/li&gt;
	&lt;li&gt;shard&lt;/li&gt;
	&lt;li&gt;get better hardware&lt;/li&gt;
	&lt;li&gt;if your traffic is bursty, you could make your oplog large enough to hold a burst of traffic and then finish replicating to secondaries during down time&lt;/li&gt;
&lt;/ol&gt;


&lt;p&gt;Those are the options I can think of off the top of my head.&lt;/p&gt;</comment>
                            <comment id="63415" author="msz" created="Fri, 28 Oct 2011 20:15:46 +0000"  >&lt;p&gt;it&apos;s about 4MBps&lt;/p&gt;</comment>
                            <comment id="63358" author="kristina" created="Fri, 28 Oct 2011 15:48:10 +0000"  >&lt;p&gt;How much data/sec were you inserting?  (E.g., 1MB/sec.)&lt;/p&gt;</comment>
                            <comment id="33890" author="msz" created="Mon, 23 May 2011 08:18:56 +0000"  >&lt;p&gt;node1:&lt;br/&gt;
Fri May 20 12:21:51 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 254744714&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 12:24:11 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; old journal file will be removed: /var/lib/mongodb/journal/j._2&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 12:31:33 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn62&amp;#93;&lt;/span&gt; end connection 172.17.0.250:42994&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 13:32:39 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:43523 #65&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 13:34:32 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; allocating new datafile /var/lib/mongodb/testdb1testdb1/testdb1.7, filling with zeroes...&lt;br/&gt;
Fri May 20 13:34:32 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; done allocating datafile /var/lib/mongodb/testdb1/testdb1.7, size: 2047MB,  took 0.02 secs&lt;br/&gt;
Fri May 20 13:35:03 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; old journal file will be removed: /var/lib/mongodb/journal/j._3&lt;br/&gt;
Fri May 20 13:35:04 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; DR101 latency warning on journal file open 773ms&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 13:36:16 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; allocating new datafile /var/lib/mongodb/testdb2/testdb2.4, filling with zeroes...&lt;br/&gt;
Fri May 20 13:36:16 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; done allocating datafile /var/lib/mongodb/testdb2/testdb2.4, size: 1024MB,  took 0.008 secs&lt;br/&gt;
Fri May 20 13:36:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 259174265&lt;br/&gt;
Fri May 20 13:37:03 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; old journal file will be removed: /var/lib/mongodb/journal/j._4&lt;br/&gt;
Fri May 20 13:37:03 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; DR101 latency warning on journal file open 297ms&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 13:40:04 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; old journal file will be removed: /var/lib/mongodb/journal/j._5&lt;br/&gt;
Fri May 20 13:40:04 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; DR101 latency warning on journal file open 752ms&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 13:41:59 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; old journal file will be removed: /var/lib/mongodb/journal/j._6&lt;br/&gt;
Fri May 20 13:41:59 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; old journal file will be removed: /var/lib/mongodb/journal/j._7&lt;br/&gt;
Fri May 20 13:42:04 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; DR101 latency warning on journal file open 4568ms&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 13:44:16 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; allocating new datafile /var/lib/mongodb/testdb2/testdb2.5, filling with zeroes...&lt;br/&gt;
Fri May 20 13:44:16 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; done allocating datafile /var/lib/mongodb/testdb2/testdb2.5, size: 2047MB,  took 0.025 secs&lt;br/&gt;
Fri May 20 13:44:34 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; allocating new datafile /var/lib/mongodb/testdb3/testdb3.6, filling with zeroes...&lt;br/&gt;
Fri May 20 13:44:34 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; done allocating datafile /var/lib/mongodb/testdb3/testdb3.6, size: 2047MB,  took 0.02 secs&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 13:45:40 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; repl: old cursor isDead, will initiate a new one&lt;br/&gt;
Fri May 20 13:45:44 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up, at least from 172.17.0.253:27017&lt;br/&gt;
Fri May 20 13:45:44 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet our last optime : May 20 13:42:19 4dd6539b:81&lt;br/&gt;
Fri May 20 13:45:44 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet oldest at 172.17.0.253:27017 : May 20 13:42:21 4dd6539d:5b&lt;br/&gt;
Fri May 20 13:45:44 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet See &lt;a href=&quot;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&lt;/a&gt;&lt;br/&gt;
Fri May 20 13:45:44 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up&lt;br/&gt;
Fri May 20 13:45:44 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet RECOVERING&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 13:47:45 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up, at least from 172.17.0.253:27017&lt;br/&gt;
Fri May 20 13:47:45 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet our last optime : May 20 13:42:19 4dd6539b:81&lt;br/&gt;
Fri May 20 13:47:45 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet oldest at 172.17.0.253:27017 : May 20 13:44:46 4dd6542e:8d&lt;br/&gt;
Fri May 20 13:47:45 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet See &lt;a href=&quot;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&lt;/a&gt;&lt;br/&gt;
Fri May 20 13:47:45 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 13:48:57 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet member 172.17.0.252:27017 RECOVERING&lt;br/&gt;
Fri May 20 13:49:45 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet SECONDARY&lt;br/&gt;
Fri May 20 13:49:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up, at least from 172.17.0.253:27017&lt;br/&gt;
Fri May 20 13:49:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet our last optime : May 20 13:42:19 4dd6539b:81&lt;br/&gt;
Fri May 20 13:49:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet oldest at 172.17.0.253:27017 : May 20 13:46:38 4dd6549e:1a&lt;br/&gt;
Fri May 20 13:49:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet See &lt;a href=&quot;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&lt;/a&gt;&lt;br/&gt;
Fri May 20 13:49:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up&lt;br/&gt;
Fri May 20 13:49:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet RECOVERING&lt;br/&gt;
Fri May 20 13:49:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 259944405&lt;br/&gt;
Fri May 20 13:50:51 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 260003625&lt;br/&gt;
Fri May 20 13:51:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up, at least from 172.17.0.253:27017&lt;br/&gt;
Fri May 20 13:51:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet our last optime : May 20 13:42:19 4dd6539b:81&lt;br/&gt;
Fri May 20 13:51:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet oldest at 172.17.0.253:27017 : May 20 13:48:32 4dd65510:a4&lt;br/&gt;
Fri May 20 13:51:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet See &lt;a href=&quot;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&lt;/a&gt;&lt;br/&gt;
Fri May 20 13:51:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 13:53:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet SECONDARY&lt;br/&gt;
Fri May 20 13:53:47 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up, at least from 172.17.0.253:27017&lt;br/&gt;
Fri May 20 13:53:47 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet our last optime : May 20 13:42:19 4dd6539b:81&lt;br/&gt;
Fri May 20 13:53:47 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet oldest at 172.17.0.253:27017 : May 20 13:50:38 4dd6558e:74&lt;br/&gt;
Fri May 20 13:53:47 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet See &lt;a href=&quot;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&lt;/a&gt;&lt;/p&gt;


&lt;p&gt;node2:&lt;br/&gt;
Fri May 20 12:22:59 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 254769358&lt;br/&gt;
Fri May 20 12:23:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; old journal file will be removed: /var/lib/mongodb/journal/j._2&lt;br/&gt;
Fri May 20 12:23:59 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 254828508&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 12:30:58 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 255242208&lt;br/&gt;
Fri May 20 12:31:33 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn56&amp;#93;&lt;/span&gt; end connection 172.17.0.250:38757&lt;br/&gt;
Fri May 20 12:31:58 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 255301278&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 13:30:58 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 258785558&lt;br/&gt;
Fri May 20 13:32:39 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:59698 #59&lt;br/&gt;
Fri May 20 13:32:59 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 258903679&lt;br/&gt;
Fri May 20 13:34:07 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 258962809&lt;br/&gt;
Fri May 20 13:34:32 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; allocating new datafile /var/lib/mongodb/testdb/testdb.7, filling with zeroes...&lt;br/&gt;
Fri May 20 13:34:32 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; done allocating datafile /var/lib/mongodb/testdb/testdb.7, size: 2047MB,  took 0.147 secs&lt;br/&gt;
Fri May 20 13:35:02 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; old journal file will be removed: /var/lib/mongodb/journal/j._3&lt;br/&gt;
Fri May 20 13:35:03 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; DR101 latency warning on journal file open 325ms&lt;br/&gt;
Fri May 20 13:35:06 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 259022019&lt;br/&gt;
Fri May 20 13:36:01 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 259081219&lt;br/&gt;
Fri May 20 13:36:16 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; allocating new datafile /var/lib/mongodb/testdb2/testdb2.4, filling with zeroes...&lt;br/&gt;
Fri May 20 13:36:16 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; done allocating datafile /var/lib/mongodb/testdb2/testdb2.4, size: 1024MB,  took 0.029 secs&lt;br/&gt;
Fri May 20 13:36:44 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; old journal file will be removed: /var/lib/mongodb/journal/j._4&lt;br/&gt;
Fri May 20 13:37:01 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 259140399&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 13:39:21 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; old journal file will be removed: /var/lib/mongodb/journal/j._5&lt;br/&gt;
Fri May 20 13:39:22 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; DR101 latency warning on journal file open 1179ms&lt;br/&gt;
Fri May 20 13:40:04 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 259318019&lt;br/&gt;
Fri May 20 13:40:28 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; old journal file will be removed: /var/lib/mongodb/journal/j._6&lt;br/&gt;
Fri May 20 13:40:30 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; DR101 latency warning on journal file open 2139ms&lt;br/&gt;
Fri May 20 13:41:04 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 259377219&lt;br/&gt;
Fri May 20 13:42:05 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 259436419&lt;br/&gt;
Fri May 20 13:42:34 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; old journal file will be removed: /var/lib/mongodb/journal/j._7&lt;br/&gt;
Fri May 20 13:42:40 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; allocating new datafile /var/lib/mongodb/testdb2/testdb2.5, filling with zeroes...&lt;br/&gt;
Fri May 20 13:42:40 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; done allocating datafile /var/lib/mongodb/testdb2/testdb2.5, size: 2047MB,  took 0.024 secs&lt;br/&gt;
Fri May 20 13:43:07 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 259495679&lt;br/&gt;
Fri May 20 13:43:22 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; allocating new datafile /var/lib/mongodb/testdb3/testdb3.6, filling with zeroes...&lt;br/&gt;
Fri May 20 13:43:24 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; done allocating datafile /var/lib/mongodb/testdb3/testdb3.6, size: 2047MB,  took 2.12 secs&lt;br/&gt;
Fri May 20 13:44:10 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 259554919&lt;br/&gt;
Fri May 20 13:44:52 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; old journal file will be removed: /var/lib/mongodb/journal/j._8&lt;br/&gt;
Fri May 20 13:44:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; DR101 latency warning on journal file open 1677ms&lt;br/&gt;
Fri May 20 13:45:04 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; allocating new datafile /var/lib/mongodb/testdb/testdb.8, filling with zeroes...&lt;br/&gt;
Fri May 20 13:45:04 &lt;span class=&quot;error&quot;&gt;&amp;#91;FileAllocator&amp;#93;&lt;/span&gt; done allocating datafile /var/lib/mongodb/testdb/testdb.8, size: 2047MB,  took 0.104 secs&lt;br/&gt;
Fri May 20 13:45:09 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 259614139&lt;br/&gt;
Fri May 20 13:45:44 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet member 172.17.0.251:27017 RECOVERING&lt;br/&gt;
Fri May 20 13:46:09 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 259673369&lt;br/&gt;
Fri May 20 13:46:24 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; old journal file will be removed: /var/lib/mongodb/journal/j._9&lt;br/&gt;
Fri May 20 13:47:06 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 259732619&lt;br/&gt;
Fri May 20 13:47:26 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; old journal file will be removed: /var/lib/mongodb/journal/j._10&lt;br/&gt;
Fri May 20 13:47:28 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; DR101 latency warning on journal file open 2143ms&lt;br/&gt;
Fri May 20 13:48:11 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 259791849&lt;br/&gt;
Fri May 20 13:48:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; repl: old cursor isDead, will initiate a new one&lt;br/&gt;
Fri May 20 13:48:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up, at least from 172.17.0.253:27017&lt;br/&gt;
Fri May 20 13:48:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet our last optime : May 20 13:45:49 4dd6546d:7c&lt;br/&gt;
Fri May 20 13:48:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet oldest at 172.17.0.253:27017 : May 20 13:45:52 4dd65470:19&lt;br/&gt;
Fri May 20 13:48:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet See &lt;a href=&quot;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&lt;/a&gt;&lt;br/&gt;
Fri May 20 13:48:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up&lt;br/&gt;
Fri May 20 13:48:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet RECOVERING&lt;br/&gt;
Fri May 20 13:49:01 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 259851079&lt;br/&gt;
Fri May 20 13:49:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet member 172.17.0.251:27017 SECONDARY&lt;br/&gt;
Fri May 20 13:49:48 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet member 172.17.0.251:27017 RECOVERING&lt;br/&gt;
Fri May 20 13:49:58 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 259910349&lt;br/&gt;
Fri May 20 13:50:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up, at least from 172.17.0.253:27017&lt;br/&gt;
Fri May 20 13:50:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet our last optime : May 20 13:45:49 4dd6546d:7c&lt;br/&gt;
Fri May 20 13:50:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet oldest at 172.17.0.253:27017 : May 20 13:47:42 4dd654de:32&lt;br/&gt;
Fri May 20 13:50:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet See &lt;a href=&quot;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member&lt;/a&gt;&lt;br/&gt;
Fri May 20 13:50:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up&lt;br/&gt;
Fri May 20 13:51:02 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 259969589&lt;br/&gt;
Fri May 20 13:51:58 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 260028829&lt;br/&gt;
Fri May 20 13:52:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet SECONDARY&lt;br/&gt;
Fri May 20 13:52:56 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up, at least from 172.17.0.253:27017&lt;/p&gt;

&lt;p&gt;node3:&lt;br/&gt;
Fri May 20 13:32:16 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 258886326&lt;br/&gt;
Fri May 20 13:32:39 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39301 #4667&lt;br/&gt;
Fri May 20 13:32:52 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4719&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;9156beebc3680af6&quot;, key: &quot;f05d79e45ac536695b38965c18f4fbe6&quot; }
&lt;p&gt; reslen:53 162ms&lt;br/&gt;
Fri May 20 13:32:52 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4718&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: { count: &quot;fs.chunks&quot;, query: {} } reslen:64 136ms&lt;br/&gt;
Fri May 20 13:32:52 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4720&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;79c80f1f192289d8&quot;, key: &quot;b22354b3db7f53f0eef75f6176241600&quot; }
&lt;p&gt; reslen:53 158ms&lt;br/&gt;
Fri May 20 13:32:52 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4722&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;80960eade10ce835&quot;, key: &quot;7afc048666bb34e8b34495e7ac683931&quot; }
&lt;p&gt; reslen:53 136ms&lt;br/&gt;
Fri May 20 13:32:52 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4721&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;5dcfc9d85982e507&quot;, key: &quot;811d6fdc4bdfe1b8c693f80af88b6c06&quot; }
&lt;p&gt; reslen:53 164ms&lt;br/&gt;
Fri May 20 13:32:52 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39492 #4723&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39556 #4752&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4740&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: { count: &quot;fs.files&quot;, query: {} } reslen:64 140ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4742&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;5680e8e7b9b0ad83&quot;, key: &quot;d970f7182b3e832db62e93d1f6521fba&quot; }
&lt;p&gt; reslen:53 136ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4744&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;e61df48abcf6cc87&quot;, key: &quot;75894067cad8b79a3a2196c822cde8cc&quot; }
&lt;p&gt; reslen:53 122ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4746&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;5d0befc966275ff1&quot;, key: &quot;b38c359cea6346e25712717f42f32ded&quot; }
&lt;p&gt; reslen:53 109ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4748&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;74ed85488bacbd59&quot;, key: &quot;2a8b9f617aa816686bda73fa7c6e313e&quot; }
&lt;p&gt; reslen:53 105ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4741&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;44d9ce89507a11c4&quot;, key: &quot;0355d84782c3457e32c900134bb7ff60&quot; }
&lt;p&gt; reslen:53 143ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4743&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;fe9947d1ec119e23&quot;, key: &quot;dac1949db953d1dfe5de8e36d5d62d0d&quot; }
&lt;p&gt; reslen:53 132ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4747&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;febb6271115d2dc5&quot;, key: &quot;9c28f6059eed554588257d7b5ba00f6e&quot; }
&lt;p&gt; reslen:53 111ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4745&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;62bfb1f5aed487cb&quot;, key: &quot;5adbf71eabdc637cbd1ae82e7b23bdf6&quot; }
&lt;p&gt; reslen:53 119ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39558 #4753&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39591 #4768&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39594 #4769&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4757&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;fa6e3dc6ab9f3e78&quot;, key: &quot;63ec868cd12362e820ca910fe6f5a501&quot; }
&lt;p&gt; reslen:53 108ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39595 #4770&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4758&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;dfb155cabd9ff2b9&quot;, key: &quot;750a64cb09635ab453afb0ca4611c683&quot; }
&lt;p&gt; reslen:53 110ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39596 #4771&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39639 #4799&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4779&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: { count: &quot;fs.files&quot;, query: {} } reslen:64 119ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4778&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: { count: &quot;fs.chunks&quot;, query: {} } reslen:64 110ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4780&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;948db4468239f928&quot;, key: &quot;312ac0375eecd3b468894ef13bba4919&quot; }
&lt;p&gt; reslen:53 111ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4781&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;dc0dd04510d0d1b6&quot;, key: &quot;8f59b0936285f160e92e3e010f873647&quot; }
&lt;p&gt; reslen:53 117ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39641 #4800&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39688 #4827&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4804&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: { count: &quot;fs.files&quot;, query: {} } reslen:64 167ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4802&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: { count: &quot;fs.chunks&quot;, query: {} } reslen:64 161ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4803&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: { count: &quot;fs.chunks&quot;, query: {} } reslen:64 161ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4807&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;7f66727043a7cc71&quot;, key: &quot;ee726ba51048c766ffa105d0db4140f5&quot; }
&lt;p&gt; reslen:53 165ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4808&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;9b5fd3a766079d69&quot;, key: &quot;548ff19cf5b130784b1e2e0f527e0bc9&quot; }
&lt;p&gt; reslen:53 165ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4810&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;22f3422890583eaa&quot;, key: &quot;0ababdd06adafd92b28cfad7018c9c2d&quot; }
&lt;p&gt; reslen:53 156ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4811&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;473c89a800b2f51a&quot;, key: &quot;c64b7a39d2b6bc3fd9f2a65eac087903&quot; }
&lt;p&gt; reslen:53 145ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4812&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;2216508136eecce3&quot;, key: &quot;804ccdab9c57f0148c6a2ad31f6e111b&quot; }
&lt;p&gt; reslen:53 133ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39690 #4828&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4805&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;fdbb98cb6e3556ed&quot;, key: &quot;6cc9b96d827904de4c8efa26f68bff59&quot; }
&lt;p&gt; reslen:53 182ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4806&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;c6a6eba17b913561&quot;, key: &quot;2a3fc8b7ae04f08eda5d002864ea14bd&quot; }
&lt;p&gt; reslen:53 175ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4809&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;9d7dd9c2118bb953&quot;, key: &quot;9c58d4cf775f354aacc2a3b2fc96c080&quot; }
&lt;p&gt; reslen:53 172ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4814&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;fa25bb9465811b7b&quot;, key: &quot;ce968c393449aee910bc0cc86a06e134&quot; }
&lt;p&gt; reslen:53 102ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4813&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;9fb768702b4f6f2f&quot;, key: &quot;ebd53d41c6e162ae2246359ce31a1238&quot; }
&lt;p&gt; reslen:53 119ms&lt;br/&gt;
Fri May 20 13:32:53 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39695 #4829&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4830&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: { count: &quot;fs.files&quot;, query: {} } reslen:64 247ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4831&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;26639b88a2190f7e&quot;, key: &quot;ca4e3e851eaa7e44f44593dad84e800e&quot; }
&lt;p&gt; reslen:53 247ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4829&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: { count: &quot;fs.files&quot;, query: {} } reslen:64 225ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39772 #4872&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4833&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;672476a875513e40&quot;, key: &quot;5774ee26dfb09f880008fb9174a88891&quot; }
&lt;p&gt; reslen:53 210ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4832&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;b044c1f31fc53aad&quot;, key: &quot;2c0315855b5914b770e3df43ac346e3a&quot; }
&lt;p&gt; reslen:53 209ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4834&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;1f0429f41f116c90&quot;, key: &quot;2b0336f95bc520415cab633e73e3cdfe&quot; }
&lt;p&gt; reslen:53 199ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4840&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;39d8f9ae977dbd3f&quot;, key: &quot;d2862b7fa09c50a9d1e932b4797e89bc&quot; }
&lt;p&gt; reslen:53 162ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4841&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;a098b4bf5ba5d6ca&quot;, key: &quot;7fb88272aec1f4c3d3fa7a3fdc7c8bb3&quot; }
&lt;p&gt; reslen:53 161ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4842&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;e88638ea3d354cab&quot;, key: &quot;d17758376a49f95e7e557dc1ddea2cbc&quot; }
&lt;p&gt; reslen:53 160ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4843&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;b7546e517ed4575a&quot;, key: &quot;cbbfbfc3a5a1651a183d5561797b767f&quot; }
&lt;p&gt; reslen:53 152ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4844&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;c9d7620dcb5dce28&quot;, key: &quot;5fe8086f11f8bf64d296df67f5ab745c&quot; }
&lt;p&gt; reslen:53 130ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4845&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;d2eeeba2f288ffaa&quot;, key: &quot;caec52ce6c8d998005564842e9be1c5f&quot; }
&lt;p&gt; reslen:53 127ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4846&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;335e1bd82d3e738d&quot;, key: &quot;cd6a84fd387706d2cfde62835ada3a77&quot; }
&lt;p&gt; reslen:53 126ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4847&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;d1d975169d4a14f6&quot;, key: &quot;d2381421cd39abaa0459de75155acbda&quot; }
&lt;p&gt; reslen:53 118ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4848&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;9fa65c5d286c734c&quot;, key: &quot;3c48651785028f24b5ebcf0b0937d371&quot; }
&lt;p&gt; reslen:53 112ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4849&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;799616d557934134&quot;, key: &quot;3477bb11006d243c172ea1c58093292e&quot; }
&lt;p&gt; reslen:53 109ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4852&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;f17ea0185206375e&quot;, key: &quot;81ff72cc7e2c03923d45f4914014d875&quot; }
&lt;p&gt; reslen:53 109ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4850&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;67bd221b12fd48b9&quot;, key: &quot;06bc602e02fa4a0e8b74eca613e66170&quot; }
&lt;p&gt; reslen:53 108ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4837&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;ac2e0bd33be611ca&quot;, key: &quot;a474826ea23bde1efd7a561724b172dc&quot; }
&lt;p&gt; reslen:53 178ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4838&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;f074bd917f994b5e&quot;, key: &quot;ca52ec6269d1b1d924a8b2f15adb8b86&quot; }
&lt;p&gt; reslen:53 177ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4839&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;8d86715d158af479&quot;, key: &quot;ff422af4a5737d192743b25e0348be72&quot; }
&lt;p&gt; reslen:53 172ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4851&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;a6b8380f5739f316&quot;, key: &quot;788e64005086a0044a6e829ff9174f37&quot; }
&lt;p&gt; reslen:53 107ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4835&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;23b30b26a2591a6c&quot;, key: &quot;2028f27e6ffffa6a80096178ae0fc226&quot; }
&lt;p&gt; reslen:53 206ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4836&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;8fd7cf9fb8ad02b6&quot;, key: &quot;7fba9f1c35cf52c3dad2e341b91946ce&quot; }
&lt;p&gt; reslen:53 194ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39776 #4873&lt;br/&gt;
...&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4873&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: { count: &quot;fs.chunks&quot;, query: {} } reslen:64 180ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4878&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;a85c997f84cee009&quot;, key: &quot;e1c3a62454210b018586ecab5880f88d&quot; }
&lt;p&gt; reslen:53 115ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4877&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;f636ab4873b7e4a4&quot;, key: &quot;95b6f31157a3846f403b69ec08b9e2e4&quot; }
&lt;p&gt; reslen:53 114ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4880&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;77967526ae648b&quot;, key: &quot;2cfa685ddcb6099a10286a3cbe24039c&quot; }
&lt;p&gt; reslen:53 102ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39819 #4900&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39820 #4901&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4875&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;4033e651e24f7ddc&quot;, key: &quot;2d27281d08d0841c4ca4863f44e0c084&quot; }
&lt;p&gt; reslen:53 158ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4876&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;e89e553e4f985dcf&quot;, key: &quot;18d142089eefcf907d1ea64603b2f06a&quot; }
&lt;p&gt; reslen:53 144ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4879&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;1d929df10ec4ce4b&quot;, key: &quot;6d7f6db0ef275f1c8cee30858d8f72b2&quot; }
&lt;p&gt; reslen:53 112ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 172.17.0.250:39821 #4902&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4902&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: { count: &quot;fs.chunks&quot;, query: {} } reslen:64 207ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4904&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;c383687d31b8981c&quot;, key: &quot;29a522b7237d4dc358acbae80f89931d&quot; }
&lt;p&gt; reslen:53 188ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4906&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;e247d27c483d9179&quot;, key: &quot;e59237acf3f9ed6a70bcb99ede316dcb&quot; }
&lt;p&gt; reslen:53 161ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4907&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;ea60de69e87d573c&quot;, key: &quot;846237912c3e8f4e5dfd1cab2e8ca542&quot; }
&lt;p&gt; reslen:53 156ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4908&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;1fd5272d84880044&quot;, key: &quot;82f40143eee6098a993b034910d73d3e&quot; }
&lt;p&gt; reslen:53 155ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4909&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;ee6e6273b8430dc8&quot;, key: &quot;ace8880e602725e4113da574d8e8b541&quot; }
&lt;p&gt; reslen:53 143ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4910&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;5c84c296377034c&quot;, key: &quot;be1ce506dcbe837b0c2cc0c51dcdefca&quot; }
&lt;p&gt; reslen:53 138ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4911&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;e59a410b3dfb2bfb&quot;, key: &quot;aa9b11f70312c8fbf5d1c3ebd85bb818&quot; }
&lt;p&gt; reslen:53 130ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4913&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;cbb3e74d174bf4b9&quot;, key: &quot;b3f38f8a245a9e83c1a5cbf742fe88aa&quot; }
&lt;p&gt; reslen:53 123ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4912&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;e6a4cb36a82b03e0&quot;, key: &quot;eeeb065efddf719266e120b21f0f50ef&quot; }
&lt;p&gt; reslen:53 122ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4914&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;6924109203235521&quot;, key: &quot;162d31fb56c91d1d0d4a3548b3367a7a&quot; }
&lt;p&gt; reslen:53 121ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4915&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;1ccf6687ed228b40&quot;, key: &quot;1aeb9a6272970c4b5f9361a0e8cf7101&quot; }
&lt;p&gt; reslen:53 116ms&lt;br/&gt;
Fri May 20 13:32:54 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4916&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: &lt;/p&gt;
{ authenticate: 1, user: &quot;dbuser&quot;, nonce: &quot;a55f49d5c7432a&quot;, key: &quot;b08f7a6c5c9466d5897bab27d957b13b&quot; }
&lt;p&gt; reslen:53 106ms&lt;br/&gt;
Fri May 20 13:35:02 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn4732&amp;#93;&lt;/span&gt; query testdb3.$cmd ntoreturn:1 command: { count: &quot;fs.chunks&quot;, query: {} } reslen:64 2415ms&lt;/p&gt;</comment>
                            <comment id="33889" author="msz" created="Mon, 23 May 2011 08:10:42 +0000"  >&lt;p&gt;node1:RECOVERING&amp;gt; db.oplog.rs.stats()&lt;br/&gt;
{&lt;br/&gt;
        &quot;ns&quot; : &quot;local.oplog.rs&quot;,&lt;br/&gt;
        &quot;count&quot; : 12234,&lt;br/&gt;
        &quot;size&quot; : 1562337904,&lt;br/&gt;
        &quot;avgObjSize&quot; : 127704.5859081249,&lt;br/&gt;
        &quot;storageSize&quot; : 1562543616,&lt;br/&gt;
        &quot;numExtents&quot; : 1,&lt;br/&gt;
        &quot;nindexes&quot; : 0,&lt;br/&gt;
        &quot;lastExtentSize&quot; : 1562543616,&lt;br/&gt;
        &quot;paddingFactor&quot; : 1,&lt;br/&gt;
        &quot;flags&quot; : 0,&lt;br/&gt;
        &quot;totalIndexSize&quot; : 0,&lt;br/&gt;
        &quot;indexSizes&quot; : {&lt;/p&gt;

&lt;p&gt;        },&lt;br/&gt;
        &quot;capped&quot; : 1,&lt;br/&gt;
        &quot;max&quot; : 2147483647,&lt;br/&gt;
        &quot;ok&quot; : 1&lt;br/&gt;
}&lt;/p&gt;


&lt;p&gt;node2:PRIMARY&amp;gt; db.oplog.rs.stats()&lt;br/&gt;
{&lt;br/&gt;
        &quot;ns&quot; : &quot;local.oplog.rs&quot;,&lt;br/&gt;
        &quot;count&quot; : 138827,&lt;br/&gt;
        &quot;size&quot; : 1410229740,&lt;br/&gt;
        &quot;avgObjSize&quot; : 10158.180613281278,&lt;br/&gt;
        &quot;storageSize&quot; : 1412488192,&lt;br/&gt;
        &quot;numExtents&quot; : 1,&lt;br/&gt;
        &quot;nindexes&quot; : 0,&lt;br/&gt;
        &quot;lastExtentSize&quot; : 1412488192,&lt;br/&gt;
        &quot;paddingFactor&quot; : 1,&lt;br/&gt;
        &quot;flags&quot; : 0,&lt;br/&gt;
        &quot;totalIndexSize&quot; : 0,&lt;br/&gt;
        &quot;indexSizes&quot; : {&lt;/p&gt;

&lt;p&gt;        },&lt;br/&gt;
        &quot;capped&quot; : 1,&lt;br/&gt;
        &quot;max&quot; : 2147483647,&lt;br/&gt;
        &quot;ok&quot; : 1&lt;br/&gt;
}&lt;/p&gt;

&lt;p&gt;node3:ROLLBACK&amp;gt; db.oplog.rs.stats();&lt;br/&gt;
{&lt;br/&gt;
        &quot;ns&quot; : &quot;local.oplog.rs&quot;,&lt;br/&gt;
        &quot;count&quot; : 60511,&lt;br/&gt;
        &quot;size&quot; : 1572346276,&lt;br/&gt;
        &quot;avgObjSize&quot; : 25984.470195501643,&lt;br/&gt;
        &quot;storageSize&quot; : 1573554432,&lt;br/&gt;
        &quot;numExtents&quot; : 1,&lt;br/&gt;
        &quot;nindexes&quot; : 0,&lt;br/&gt;
        &quot;lastExtentSize&quot; : 1573554432,&lt;br/&gt;
        &quot;paddingFactor&quot; : 1,&lt;br/&gt;
        &quot;flags&quot; : 0,&lt;br/&gt;
        &quot;totalIndexSize&quot; : 0,&lt;br/&gt;
        &quot;indexSizes&quot; : {&lt;/p&gt;

&lt;p&gt;        },&lt;br/&gt;
        &quot;capped&quot; : 1,&lt;br/&gt;
        &quot;max&quot; : 2147483647,&lt;br/&gt;
        &quot;ok&quot; : 1&lt;br/&gt;
}&lt;/p&gt;</comment>
                            <comment id="33683" author="kristina" created="Fri, 20 May 2011 16:14:41 +0000"  >&lt;p&gt;Do you have the actual db logs (not the log from the web ui) from between 12:29 and 13:48?&lt;/p&gt;

&lt;p&gt;      12:29:18 &lt;span class=&quot;error&quot;&gt;&amp;#91;rs Manager&amp;#93;&lt;/span&gt; replSet not trying to elect self as responded yea to someone else recently&lt;br/&gt;
           12:29:19 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet info 172.17.0.253:27017 is up&lt;br/&gt;
Fri May 20 13:45:44 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet member 172.17.0.251:27017 RECOVERING&lt;br/&gt;
           13:48:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet error RS102 too stale to catch up, at least from 172.17.0.253:27017&lt;br/&gt;
           13:48:55 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet our last optime : May 20 13:45:49 4dd6546d:7c &lt;/p&gt;

&lt;p&gt;Also, can you run:&lt;/p&gt;

&lt;p&gt;&amp;gt; use local&lt;br/&gt;
&amp;gt; db.oplog.rs.stats()&lt;/p&gt;

&lt;p&gt;on all three nodes and send the output?&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>8.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Fri, 20 May 2011 16:14:41 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        12 years, 14 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ian@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            12 years, 14 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10000" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Old_Backport</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10000"><![CDATA[No]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10020"><![CDATA[Linux]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>kristina</customfieldvalue>
            <customfieldvalue>msz</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrozfj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrifif:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>21165</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hs386f:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>