<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:02:02 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-3087] &quot;Assertion: 13312:replSet error : logOp() but not primary?&quot; and &quot;com.mongodb.MongoException: not talking to master and retries used up&quot;</title>
                <link>https://jira.mongodb.org/browse/SERVER-3087</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Something (I don&apos;t know what yet) caused one of my shards to fail over. When I look at the logs on the old master, I see this:&lt;/p&gt;

&lt;p&gt;Thu May 12 15:47:22 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn21060&amp;#93;&lt;/span&gt; remove example.userEventsJournal  query: &lt;/p&gt;
{ _id: ObjectId(&apos;4d56ea01fa00a03b38c17fde&apos;) }
&lt;p&gt; 232ms&lt;br/&gt;
Thu May 12 15:47:23 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn21060&amp;#93;&lt;/span&gt; remove example.userEventsJournal  query: &lt;/p&gt;
{ _id: ObjectId(&apos;4d56ea0a04e235872bd9e29e&apos;) }
&lt;p&gt; 291ms&lt;br/&gt;
Thu May 12 15:47:25 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn20802&amp;#93;&lt;/span&gt; query example.$cmd ntoreturn:1 command: { distinct: &quot;cookbook&quot;, key: &quot;cookbookId&quot;, query: {} } reslen:1445 1401ms&lt;br/&gt;
Thu May 12 15:47:25 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn20841&amp;#93;&lt;/span&gt; query example.$cmd ntoreturn:1 command: { distinct: &quot;cookbook&quot;, key: &quot;cookbookId&quot;, query: {} } reslen:1445 664ms&lt;br/&gt;
Thu May 12 15:47:28 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 120976684&lt;br/&gt;
Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn21062&amp;#93;&lt;/span&gt; getmore local.oplog.rs cid:3926093491549750074 getMore: { ts: &lt;/p&gt;
{ $gte: new Date(5605849438531616885) }
&lt;p&gt; }  bytes:789 nreturned:3 12119ms&lt;br/&gt;
Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet member mongodb05.example.com:27018 PRIMARY&lt;br/&gt;
Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn21035&amp;#93;&lt;/span&gt; Assertion: 13312:replSet error : logOp() but not primary?&lt;br/&gt;
0x55f1b9 0x712a31 0x7108f2 0x668fa1 0x66ba60 0x75814f 0x75aae1 0x8a75ee 0x8ba510 0x7fb642df69ca 0x7fb6423a570d &lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo11msgassertedEiPKc+0x129) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x55f1b9&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod() &lt;span class=&quot;error&quot;&gt;&amp;#91;0x712a31&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo5logOpEPKcS1_RKNS_7BSONObjEPS2_Pb+0x42) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7108f2&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo14_updateObjectsEbPKcRKNS_7BSONObjES2_bbbRNS_7OpDebugEPNS_11RemoveSaverE+0x1131) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x668fa1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo13updateObjectsEPKcRKNS_7BSONObjES2_bbbRNS_7OpDebugE+0x130) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x66ba60&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo14receivedUpdateERNS_7MessageERNS_5CurOpE+0x47f) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x75814f&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo16assembleResponseERNS_7MessageERNS_10DbResponseERKNS_8SockAddrE+0x1941) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x75aae1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo10connThreadEPNS_13MessagingPortE+0x21e) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x8a75ee&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(thread_proxy+0x80) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x8ba510&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/libpthread.so.0(+0x69ca) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fb642df69ca&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/libc.so.6(clone+0x6d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fb6423a570d&amp;#93;&lt;/span&gt;&lt;br/&gt;
Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn21035&amp;#93;&lt;/span&gt; update example.stats  query: &lt;/p&gt;
{ entityId: &quot;hal_livingSocial_DFA03&quot;, bucketId: &quot;groups&quot;, variationId: &quot;_default&quot; }
&lt;p&gt; exception 13312 replSet error : logOp() but not primary? 1ms&lt;br/&gt;
Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn20999&amp;#93;&lt;/span&gt; Assertion: 13312:replSet error : logOp() but not primary?&lt;br/&gt;
0x55f1b9 0x712a31 0x7108f2 0x668fa1 0x66ba60 0x75814f 0x75aae1 0x8a75ee 0x8ba510 0x7fb642df69ca 0x7fb6423a570d &lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo11msgassertedEiPKc+0x129) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x55f1b9&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod() &lt;span class=&quot;error&quot;&gt;&amp;#91;0x712a31&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo5logOpEPKcS1_RKNS_7BSONObjEPS2_Pb+0x42) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7108f2&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo14_updateObjectsEbPKcRKNS_7BSONObjES2_bbbRNS_7OpDebugEPNS_11RemoveSaverE+0x1131) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x668fa1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo13updateObjectsEPKcRKNS_7BSONObjES2_bbbRNS_7OpDebugE+0x130) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x66ba60&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo14receivedUpdateERNS_7MessageERNS_5CurOpE+0x47f) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x75814f&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo16assembleResponseERNS_7MessageERNS_10DbResponseERKNS_8SockAddrE+0x1941) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x75aae1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo10connThreadEPNS_13MessagingPortE+0x21e) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x8a75ee&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(thread_proxy+0x80) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x8ba510&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/libpthread.so.0(+0x69ca) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fb642df69ca&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib/libc.so.6(clone+0x6d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x7fb6423a570d&amp;#93;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;There are many more of those &quot;logOp() but not primary?&quot; messages after this. Followed by several messages like this:&lt;/p&gt;

&lt;p&gt;Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn21003&amp;#93;&lt;/span&gt; update example.stats  query: &lt;/p&gt;
{ entityId: &quot;hal_livingSocial_DFA03&quot;, bucketId: &quot;groups&quot;, variationId: &quot;_default&quot; }
&lt;p&gt; exception 13312 replSet error : logOp() but not primary? 1ms&lt;br/&gt;
Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn21033&amp;#93;&lt;/span&gt; assertion 13435 not master and slaveok=false ns:example.ads query:&lt;/p&gt;
{ adId: &quot;hal_livingSocial_cookbook_dfa_winning02&quot; }
&lt;p&gt;Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn21033&amp;#93;&lt;/span&gt;  ntoskip:0 ntoreturn:-1&lt;br/&gt;
Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn21024&amp;#93;&lt;/span&gt; assertion 13435 not master and slaveok=false ns:example.cookbook query:&lt;/p&gt;
{ cookbookId: &quot;livingSocialBaselineVariation07&quot; }
&lt;p&gt;Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn21024&amp;#93;&lt;/span&gt;  ntoskip:0 ntoreturn:-1&lt;br/&gt;
Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn21012&amp;#93;&lt;/span&gt; assertion 13435 not master and slaveok=false ns:example.cookbook query:&lt;/p&gt;
{ cookbookId: &quot;livingSocialBaselineVariation08&quot; }
&lt;p&gt;Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn21012&amp;#93;&lt;/span&gt;  ntoskip:0 ntoreturn:-1&lt;br/&gt;
Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn21018&amp;#93;&lt;/span&gt; assertion 13435 not master and slaveok=false ns:example.cookbook query:&lt;/p&gt;
{ cookbookId: &quot;livingSocialBaselineVariation07&quot; }
&lt;p&gt;Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn21018&amp;#93;&lt;/span&gt;  ntoskip:0 ntoreturn:-1&lt;br/&gt;
Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn20992&amp;#93;&lt;/span&gt; assertion 13435 not master and slaveok=false ns:example.cookbook query:&lt;/p&gt;
{ cookbookId: &quot;livingSocialBaselineVariation08&quot; }
&lt;p&gt;Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn20992&amp;#93;&lt;/span&gt;  ntoskip:0 ntoreturn:-1Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn20994&amp;#93;&lt;/span&gt; assertion 13435 not master and slaveok=false ns:example.cookbook query:&lt;/p&gt;
{ cookbookId: &quot;livingSocialBaselineVariation08&quot; }
&lt;p&gt;Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn20994&amp;#93;&lt;/span&gt;  ntoskip:0 ntoreturn:-1&lt;/p&gt;

&lt;p&gt;Then this:&lt;/p&gt;

&lt;p&gt;Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet our last op time written: May 12 15:47:29:10&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replset source&apos;s GTE: May 12 15:47:41:1&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet rollback 0&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet rollback 1&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet rollback 2 FindCommonPoint&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet info rollback our last optime:   May 12 15:47:29:10&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet info rollback their last optime: May 12 15:47:42:a&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet info rollback diff in end of log times: -13 secondsThu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet rollback found matching events at May 12 15:47:29:d&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet rollback findcommonpoint scanned : 38&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet replSet rollback 3 fixup&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet rollback 3.5&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet rollback 4 n:3&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet minvalid=May 12 15:47:42 4dcc011e:a&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet rollback 4.6&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet rollback 4.7&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet rollback 5 d:0 u:3&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet rollback 6&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet rollback 7&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet rollback done&lt;br/&gt;
Thu May 12 15:47:43 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 120982480&lt;/p&gt;

&lt;p&gt;And finally:&lt;/p&gt;

&lt;p&gt;Thu May 12 15:48:24 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 121000882&lt;br/&gt;
Thu May 12 15:48:25 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; DBClientCursor::init call() failed&lt;br/&gt;
Thu May 12 15:48:25 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet info mongodb05.example.com:27018 is down (or slow to respond): DBClientBase::findOne: transport error: mongodb05.example.com:27018 query: &lt;/p&gt;
{ replSetHeartbeat: &quot;2&quot;, v: 7, pv: 1, checkEmpty: false, from: &quot;mongodb04.example.com:27018&quot; }
&lt;p&gt;Thu May 12 15:48:25 &lt;span class=&quot;error&quot;&gt;&amp;#91;rs Manager&amp;#93;&lt;/span&gt; replSet info electSelf 0&lt;br/&gt;
Thu May 12 15:48:25 &lt;span class=&quot;error&quot;&gt;&amp;#91;rs Manager&amp;#93;&lt;/span&gt; replSet couldn&apos;t elect self, only received -9999 votes&lt;br/&gt;
Thu May 12 15:48:29 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet info mongodb05.example.com:27018 is up&lt;br/&gt;
Thu May 12 15:49:10 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn21155&amp;#93;&lt;/span&gt; query local.$cmd ntoreturn:1 command: &lt;/p&gt;
{ collstats: &quot;oplog.rs&quot; }
&lt;p&gt; reslen:284 3638ms&lt;br/&gt;
Thu May 12 15:49:33 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 121030916&lt;br/&gt;
Thu May 12 15:50:20 &lt;span class=&quot;error&quot;&gt;&amp;#91;dur&amp;#93;&lt;/span&gt; lsn set 121060941&lt;/p&gt;

&lt;p&gt;On the &lt;b&gt;new&lt;/b&gt; master, I see this:&lt;/p&gt;

&lt;p&gt;Thu May 12 15:47:40 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; DBClientCursor::init call() failed&lt;br/&gt;
Thu May 12 15:47:40 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet info mongodb04.socialmedia.com:27018 is down (or slow to respond): DBClientBase::findOne: transport error: mongodb04.socialmedia.com:27018 query: &lt;/p&gt;
{ replSetHeartbeat: &quot;2&quot;, v: 7, pv: 1, checkEmpty: false, from: &quot;mongodb05.socialmedia.com:27018&quot; }
&lt;p&gt;Thu May 12 15:47:40 &lt;span class=&quot;error&quot;&gt;&amp;#91;rs Manager&amp;#93;&lt;/span&gt; replSet info electSelf 1&lt;br/&gt;
Thu May 12 15:47:40 &lt;span class=&quot;error&quot;&gt;&amp;#91;rs Manager&amp;#93;&lt;/span&gt; replSet PRIMARY&lt;br/&gt;
Thu May 12 15:47:41 &lt;span class=&quot;error&quot;&gt;&amp;#91;replica set sync&amp;#93;&lt;/span&gt; replSet stopping syncTail we are now primary&lt;br/&gt;
Thu May 12 15:47:42 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet info mongodb04.socialmedia.com:27018 is up&lt;br/&gt;
Thu May 12 15:47:42 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet member mongodb04.socialmedia.com:27018 RECOVERING&lt;br/&gt;
Thu May 12 15:47:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn52890&amp;#93;&lt;/span&gt; remove socialmedia.userEventsJournal  query: &lt;/p&gt;
{ _id: ObjectId(&apos;4d56eb456655aa982210b2b9&apos;) }
&lt;p&gt; 115ms&lt;br/&gt;
Thu May 12 15:47:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplSetHealthPollTask&amp;#93;&lt;/span&gt; replSet member mongodb04.socialmedia.com:27018 SECONDARY&lt;/p&gt;

&lt;p&gt;So failover appears to have completed successfully. Though now I&apos;m seeing exceptions like this in my app:&lt;/p&gt;

&lt;p&gt;ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes: com.mongodb.MongoException: not talking to master and retries used up&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at com.mongodb.DBTCPConnector.call(DBTCPConnector.java:227)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at com.mongodb.DBTCPConnector.call(DBTCPConnector.java:229)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at com.mongodb.DBTCPConnector.call(DBTCPConnector.java:229)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at com.mongodb.DBApiLayer$MyCollection.__find(DBApiLayer.java:295)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at com.mongodb.DBCursor._check(DBCursor.java:354)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at com.mongodb.DBCursor._hasNext(DBCursor.java:484)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at com.mongodb.DBCursor.hasNext(DBCursor.java:509)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.persistence.mongo.RealDatabaseCollection$$anon$1.hasNext(RealMongoImplementation.scala:82)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at scala.collection.IterableLike$class.isEmpty(IterableLike.scala:92)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.persistence.mongo.IterableViewImpl.isEmpty(RealMongoImplementation.scala:122)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at scala.collection.TraversableLike$class.headOption(TraversableLike.scala:483)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.persistence.mongo.IterableViewImpl.headOption(RealMongoImplementation.scala:122)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.persistence.mongo.QueryBehaviours$SelectOneQueryBehaviour$class.query(QueryBehaviours.scala:128)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.persistence.mongo.MongoSelectOneQuery.query(MongoQuery.scala:71)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.persistence.mongo.MongoSelectOneQuery.query(MongoQuery.scala:71)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.persistence.mongo.QueryBehaviours$MongoQueryBehaviour$class.apply(QueryBehaviours.scala:16)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.persistence.mongo.MongoSelectOneQuery.apply(MongoQuery.scala:71)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.persistence.mongo.MongoSelectOneQuery.apply(MongoQuery.scala:71)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.persistence.mongo.MongoDatabase$$anonfun$blueeyes$persistence$mongo$MongoDatabase$$mongoActor$1.apply(Mongo.scala:46)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.persistence.mongo.MongoDatabase$$anonfun$blueeyes$persistence$mongo$MongoDatabase$$mongoActor$1.apply(Mongo.scala:43)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.concurrent.ActorExecutionStrategySequential$$anon$14$$anonfun$submit$1.apply(Actor.scala:17)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.concurrent.Future$$anonfun$deliver$1.apply(Future.scala:50)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.concurrent.Future$$anonfun$deliver$1.apply(Future.scala:46)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.concurrent.ReadWriteLock$class.writeLock(ReadWriteLock.scala:10)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.concurrent.Future$$anon$1.writeLock(Future.scala:22)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.concurrent.Future.deliver(Future.scala:45)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.concurrent.ActorExecutionStrategySequential$$anon$14.submit(Actor.scala:17)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.concurrent.ActorImplementationMultiThreaded$StrategyWorker$$anonfun$run$1.apply$mcV$sp(Actor.scala:195)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.concurrent.ActorImplementationMultiThreaded$ActorContext$.withActorFn(Actor.scala:230)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     at blueeyes.concurrent.ActorImplementationMultiThreaded$StrategyWorker.run(Actor.scala:189)&lt;br/&gt;
ERR &lt;span class=&quot;error&quot;&gt;&amp;#91;20110512-15:47:43.401&amp;#93;&lt;/span&gt; blueeyes:     (...more...)&lt;/p&gt;

&lt;p&gt;I do not see any errors on the mongos.log around this time:&lt;/p&gt;

&lt;p&gt;Thu May 12 15:47:16 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; skipping balancing round because balancing is disabled&lt;br/&gt;
Thu May 12 15:47:26 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicaSetMonitorWatcher&amp;#93;&lt;/span&gt; ReplicaSetMonitor::_checkConnection: mongodb10.example.com:27018 &lt;/p&gt;
{ setName: &quot;4&quot;, ismaster: true, secondary: false, hosts: [ &quot;mongodb10.example.com:27018&quot;, &quot;mongodb11.example.com:27018&quot; ], arbiters: [ &quot;mongodb12.example.com:27018&quot; ], maxBsonObjectSize: 16777216, ok: 1.0 }
&lt;p&gt;checking replica set: 1&lt;br/&gt;
Thu May 12 15:47:26 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicaSetMonitorWatcher&amp;#93;&lt;/span&gt; ReplicaSetMonitor::_checkConnection: mongodb01.example.com:27018 &lt;/p&gt;
{ setName: &quot;1&quot;, ismaster: true, secondary: false, hosts: [ &quot;mongodb01.example.com:27018&quot;, &quot;mongodb02.example.com:27018&quot; ], arbiters: [ &quot;mongodb03.example.com:27018&quot; ], maxBsonObjectSize: 16777216, ok: 1.0 }
&lt;p&gt;checking replica set: 2&lt;br/&gt;
Thu May 12 15:47:26 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicaSetMonitorWatcher&amp;#93;&lt;/span&gt; ReplicaSetMonitor::_checkConnection: mongodb04.example.com:27018 &lt;/p&gt;
{ setName: &quot;2&quot;, ismaster: true, secondary: false, hosts: [ &quot;mongodb04.example.com:27018&quot;, &quot;mongodb05.example.com:27018&quot; ], arbiters: [ &quot;mongodb06.example.com:27018&quot; ], maxBsonObjectSize: 16777216, ok: 1.0 }
&lt;p&gt;checking replica set: 3&lt;br/&gt;
Thu May 12 15:47:26 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicaSetMonitorWatcher&amp;#93;&lt;/span&gt; ReplicaSetMonitor::_checkConnection: mongodb07.example.com:27018 &lt;/p&gt;
{ setName: &quot;3&quot;, ismaster: true, secondary: false, hosts: [ &quot;mongodb07.example.com:27018&quot;, &quot;mongodb08.example.com:27018&quot; ], arbiters: [ &quot;mongodb09.example.com:27018&quot; ], maxBsonObjectSize: 16777216, ok: 1.0 }
&lt;p&gt;checking replica set: 4&lt;br/&gt;
Thu May 12 15:47:33 &lt;span class=&quot;error&quot;&gt;&amp;#91;LockPinger&amp;#93;&lt;/span&gt; dist_lock pinged successfully for: appserver01:1305178845:1804289383&lt;br/&gt;
Thu May 12 15:47:36 &lt;span class=&quot;error&quot;&gt;&amp;#91;WriteBackListener&amp;#93;&lt;/span&gt; writebacklisten result: &lt;/p&gt;
{ noop: true, ok: 1.0 }
&lt;p&gt;Thu May 12 15:47:36 &lt;span class=&quot;error&quot;&gt;&amp;#91;WriteBackListener&amp;#93;&lt;/span&gt; writebacklisten result: &lt;/p&gt;
{ noop: true, ok: 1.0 }
&lt;p&gt;Thu May 12 15:47:36 &lt;span class=&quot;error&quot;&gt;&amp;#91;WriteBackListener&amp;#93;&lt;/span&gt; writebacklisten result: &lt;/p&gt;
{ noop: true, ok: 1.0 }
&lt;p&gt;Thu May 12 15:47:36 &lt;span class=&quot;error&quot;&gt;&amp;#91;WriteBackListener&amp;#93;&lt;/span&gt; writebacklisten result: &lt;/p&gt;
{ noop: true, ok: 1.0 }
&lt;p&gt;Thu May 12 15:47:36 &lt;span class=&quot;error&quot;&gt;&amp;#91;WriteBackListener&amp;#93;&lt;/span&gt; writebacklisten result: &lt;/p&gt;
{ noop: true, ok: 1.0 }
&lt;p&gt;Thu May 12 15:47:36 &lt;span class=&quot;error&quot;&gt;&amp;#91;WriteBackListener&amp;#93;&lt;/span&gt; writebacklisten result: &lt;/p&gt;
{ noop: true, ok: 1.0 }
&lt;p&gt;Thu May 12 15:47:36 &lt;span class=&quot;error&quot;&gt;&amp;#91;WriteBackListener&amp;#93;&lt;/span&gt; writebacklisten result: &lt;/p&gt;
{ noop: true, ok: 1.0 }
&lt;p&gt;Thu May 12 15:47:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicaSetMonitorWatcher&amp;#93;&lt;/span&gt; ReplicaSetMonitor::_checkConnection: mongodb10.example.com:27018 &lt;/p&gt;
{ setName: &quot;4&quot;, ismaster: true, secondary: false, hosts: [ &quot;mongodb10.example.com:27018&quot;, &quot;mongodb11.example.com:27018&quot; ], arbiters: [ &quot;mongodb12.example.com:27018&quot; ], maxBsonObjectSize: 16777216, ok: 1.0 }
&lt;p&gt;checking replica set: 1&lt;br/&gt;
Thu May 12 15:47:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicaSetMonitorWatcher&amp;#93;&lt;/span&gt; ReplicaSetMonitor::_checkConnection: mongodb01.example.com:27018 &lt;/p&gt;
{ setName: &quot;1&quot;, ismaster: true, secondary: false, hosts: [ &quot;mongodb01.example.com:27018&quot;, &quot;mongodb02.example.com:27018&quot; ], arbiters: [ &quot;mongodb03.example.com:27018&quot; ], maxBsonObjectSize: 16777216, ok: 1.0 }
&lt;p&gt;checking replica set: 2&lt;br/&gt;
Thu May 12 15:47:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicaSetMonitorWatcher&amp;#93;&lt;/span&gt; ReplicaSetMonitor::_checkConnection: mongodb04.example.com:27018 &lt;/p&gt;
{ setName: &quot;2&quot;, ismaster: false, secondary: true, hosts: [ &quot;mongodb04.example.com:27018&quot;, &quot;mongodb05.example.com:27018&quot; ], arbiters: [ &quot;mongodb06.example.com:27018&quot; ], primary: &quot;mongodb05.example.com:27018&quot;, maxBsonObjectSize: 16777216, ok: 1.0 }
&lt;p&gt;_check : 2/mongodb05.example.com:27018,mongodb04.example.com:27018,mongodb04.example.com:27018&lt;br/&gt;
Thu May 12 15:47:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicaSetMonitorWatcher&amp;#93;&lt;/span&gt; ReplicaSetMonitor::_checkConnection: mongodb05.example.com:27018 &lt;/p&gt;
{ setName: &quot;2&quot;, ismaster: true, secondary: false, hosts: [ &quot;mongodb05.example.com:27018&quot;, &quot;mongodb04.example.com:27018&quot; ], arbiters: [ &quot;mongodb06.example.com:27018&quot; ], maxBsonObjectSize: 16777216, ok: 1.0 }
&lt;p&gt;checking replica set: 3&lt;br/&gt;
Thu May 12 15:47:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicaSetMonitorWatcher&amp;#93;&lt;/span&gt; ReplicaSetMonitor::_checkConnection: mongodb07.example.com:27018 &lt;/p&gt;
{ setName: &quot;3&quot;, ismaster: true, secondary: false, hosts: [ &quot;mongodb07.example.com:27018&quot;, &quot;mongodb08.example.com:27018&quot; ], arbiters: [ &quot;mongodb09.example.com:27018&quot; ], maxBsonObjectSize: 16777216, ok: 1.0 }
&lt;p&gt;checking replica set: 4&lt;br/&gt;
Thu May 12 15:47:46 BackgroundJob starting: ConnectBG&lt;br/&gt;
Thu May 12 15:47:46 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; skipping balancing round because balancing is disabled&lt;br/&gt;
Thu May 12 15:47:47 BackgroundJob starting: ConnectBG&lt;br/&gt;
Thu May 12 15:47:50 &lt;span class=&quot;error&quot;&gt;&amp;#91;WriteBackListener&amp;#93;&lt;/span&gt; writebacklisten result: &lt;/p&gt;
{ noop: true, ok: 1.0 }
&lt;p&gt;Thu May 12 15:48:03 &lt;span class=&quot;error&quot;&gt;&amp;#91;LockPinger&amp;#93;&lt;/span&gt; dist_lock pinged successfully for: appserver01:1305178845:1804289383&lt;br/&gt;
Thu May 12 15:48:06 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicaSetMonitorWatcher&amp;#93;&lt;/span&gt; ReplicaSetMonitor::_checkConnection: mongodb10.example.com:27018 &lt;/p&gt;
{ setName: &quot;4&quot;, ismaster: true, secondary: false, hosts: [ &quot;mongodb10.example.com:27018&quot;, &quot;mongodb11.example.com:27018&quot; ], arbiters: [ &quot;mongodb12.example.com:27018&quot; ], maxBsonObjectSize: 16777216, ok: 1.0 }
&lt;p&gt;checking replica set: 1&lt;br/&gt;
Thu May 12 15:48:06 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicaSetMonitorWatcher&amp;#93;&lt;/span&gt; ReplicaSetMonitor::_checkConnection: mongodb01.example.com:27018 &lt;/p&gt;
{ setName: &quot;1&quot;, ismaster: true, secondary: false, hosts: [ &quot;mongodb01.example.com:27018&quot;, &quot;mongodb02.example.com:27018&quot; ], arbiters: [ &quot;mongodb03.example.com:27018&quot; ], maxBsonObjectSize: 16777216, ok: 1.0 }
&lt;p&gt;checking replica set: 2&lt;br/&gt;
Thu May 12 15:48:06 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicaSetMonitorWatcher&amp;#93;&lt;/span&gt; ReplicaSetMonitor::_checkConnection: mongodb05.example.com:27018 &lt;/p&gt;
{ setName: &quot;2&quot;, ismaster: true, secondary: false, hosts: [ &quot;mongodb05.example.com:27018&quot;, &quot;mongodb04.example.com:27018&quot; ], arbiters: [ &quot;mongodb06.example.com:27018&quot; ], maxBsonObjectSize: 16777216, ok: 1.0 }
&lt;p&gt;checking replica set: 3&lt;br/&gt;
Thu May 12 15:48:06 &lt;span class=&quot;error&quot;&gt;&amp;#91;ReplicaSetMonitorWatcher&amp;#93;&lt;/span&gt; ReplicaSetMonitor::_checkConnection: mongodb07.example.com:27018 &lt;/p&gt;
{ setName: &quot;3&quot;, ismaster: true, secondary: false, hosts: [ &quot;mongodb07.example.com:27018&quot;, &quot;mongodb08.example.com:27018&quot; ], arbiters: [ &quot;mongodb09.example.com:27018&quot; ], maxBsonObjectSize: 16777216, ok: 1.0 }
&lt;p&gt;checking replica set: 4&lt;/p&gt;</description>
                <environment>Ubuntu on EC2</environment>
        <key id="16971">SERVER-3087</key>
            <summary>&quot;Assertion: 13312:replSet error : logOp() but not primary?&quot; and &quot;com.mongodb.MongoException: not talking to master and retries used up&quot;</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.mongodb.org/images/icons/priorities/major.svg">Major - P3</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="4">Incomplete</resolution>
                                        <assignee username="greg_10gen">Greg Studer</assignee>
                                    <reporter username="mconigliaro">Michael Conigliaro</reporter>
                        <labels>
                    </labels>
                <created>Thu, 12 May 2011 16:57:24 +0000</created>
                <updated>Wed, 15 Aug 2012 14:24:08 +0000</updated>
                            <resolved>Wed, 13 Jun 2012 14:04:38 +0000</resolved>
                                    <version>1.8.2</version>
                                                                        <votes>0</votes>
                                    <watches>2</watches>
                                                                                                                <comments>
                            <comment id="131753" author="greg_10gen" created="Wed, 13 Jun 2012 14:04:38 +0000"  >&lt;p&gt;Cursor causing the problem has been deprecated since 1.8 - no repro.&lt;/p&gt;</comment>
                            <comment id="113776" author="ian@10gen.com" created="Wed, 25 Apr 2012 20:58:26 +0000"  >&lt;p&gt;@michael could you let us know if you&apos;re seeing this on 2.0?&lt;/p&gt;</comment>
                            <comment id="64213" author="greg_10gen" created="Wed, 2 Nov 2011 15:03:26 +0000"  >&lt;p&gt;Are you still seeing this issue with newer mongos versions?  The SerialServer cursor has been deprecated in 2.0.  &lt;/p&gt;</comment>
                            <comment id="32402" author="greg_10gen" created="Thu, 12 May 2011 22:13:59 +0000"  >&lt;p&gt;Think there may be an issue in the way limits are propagated between mongo clients &amp;#8211; looking into it...&lt;/p&gt;

&lt;p&gt;See &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-2997&quot; title=&quot;limits do not seem to work correctly in shell when count()&amp;#39;ing $near queries&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-2997&quot;&gt;&lt;del&gt;SERVER-2997&lt;/del&gt;&lt;/a&gt; for a similar issue with this...&lt;/p&gt;</comment>
                            <comment id="32374" author="mconigliaro" created="Thu, 12 May 2011 20:28:03 +0000"  >&lt;p&gt;Quick update. nscanned on shard 1 actually makes sense because it turns out that none of the documents there match the query (a full scan would have been necessary to figure that out). I don&apos;t understand what&apos;s going on with shard 2 though. I&apos;ve been running that script directly on mongodb04 for over an hour now with no problems. If I try running it via mongos, it triggers a failover very quickly.&lt;/p&gt;</comment>
                            <comment id="32356" author="mconigliaro" created="Thu, 12 May 2011 19:17:56 +0000"  >&lt;p&gt;Here&apos;s the explain() output from the mongos. Why is nscanned so much bigger here?&lt;/p&gt;

&lt;p&gt;&amp;gt; db.userEventsJournal.find({ &quot;eventId&quot; : &lt;/p&gt;
{ $nin: [&quot;click&quot;, &quot;clickThrough&quot;, &quot;conversion&quot;, &quot;emailConversion&quot;, &quot;purchaseConversion&quot;]}
&lt;p&gt; }).limit(1).explain()&lt;br/&gt;
{&lt;br/&gt;
        &quot;clusteredType&quot; : &quot;SerialServer&quot;,&lt;br/&gt;
        &quot;shards&quot; : {&lt;br/&gt;
                &quot;1/mongodb01.example.com:27018,mongodb02.example.com:27018&quot; : [&lt;br/&gt;
                        {&lt;br/&gt;
                                &quot;cursor&quot; : &quot;BasicCursor&quot;,&lt;br/&gt;
                                &quot;nscanned&quot; : 132391,&lt;br/&gt;
                                &quot;nscannedObjects&quot; : 132391,&lt;br/&gt;
                                &quot;n&quot; : 0,&lt;br/&gt;
                                &quot;millis&quot; : 203,&lt;br/&gt;
                                &quot;nYields&quot; : 0,&lt;br/&gt;
                                &quot;nChunkSkips&quot; : 0,&lt;br/&gt;
                                &quot;isMultiKey&quot; : false,&lt;br/&gt;
                                &quot;indexOnly&quot; : false,&lt;br/&gt;
                                &quot;indexBounds&quot; : {&lt;/p&gt;

&lt;p&gt;                                }&lt;br/&gt;
                        }&lt;br/&gt;
                ],&lt;br/&gt;
                &quot;2/mongodb05.example.com:27018,mongodb04.example.com:27018&quot; : [&lt;br/&gt;
                        {&lt;br/&gt;
                                &quot;cursor&quot; : &quot;BasicCursor&quot;,&lt;br/&gt;
                                &quot;nscanned&quot; : 6985870,&lt;br/&gt;
                                &quot;nscannedObjects&quot; : 6985870,&lt;br/&gt;
                                &quot;n&quot; : 1,&lt;br/&gt;
                                &quot;millis&quot; : 24938,&lt;br/&gt;
                                &quot;nYields&quot; : 224,&lt;br/&gt;
                                &quot;nChunkSkips&quot; : 6974952,&lt;br/&gt;
                                &quot;isMultiKey&quot; : false,&lt;br/&gt;
                                &quot;indexOnly&quot; : false,&lt;br/&gt;
                                &quot;indexBounds&quot; : {&lt;/p&gt;

&lt;p&gt;                                }&lt;br/&gt;
                        }&lt;br/&gt;
                ],&lt;br/&gt;
                &quot;3/mongodb07.example.com:27018,mongodb08.example.com:27018&quot; : [&lt;br/&gt;
                        {&lt;br/&gt;
                                &quot;cursor&quot; : &quot;BasicCursor&quot;,&lt;br/&gt;
                                &quot;nscanned&quot; : 1,&lt;br/&gt;
                                &quot;nscannedObjects&quot; : 1,&lt;br/&gt;
                                &quot;n&quot; : 1,&lt;br/&gt;
                                &quot;millis&quot; : 13,&lt;br/&gt;
                                &quot;nYields&quot; : 0,&lt;br/&gt;
                                &quot;nChunkSkips&quot; : 0,&lt;br/&gt;
                                &quot;isMultiKey&quot; : false,&lt;br/&gt;
                                &quot;indexOnly&quot; : false,&lt;br/&gt;
                                &quot;indexBounds&quot; : {&lt;/p&gt;

&lt;p&gt;                                }&lt;br/&gt;
                        }&lt;br/&gt;
                ],&lt;br/&gt;
                &quot;4/mongodb10.example.com:27018,mongodb11.example.com:27018&quot; : [&lt;br/&gt;
                        {&lt;br/&gt;
                                &quot;cursor&quot; : &quot;BasicCursor&quot;,&lt;br/&gt;
                                &quot;nscanned&quot; : 1,&lt;br/&gt;
                                &quot;nscannedObjects&quot; : 1,&lt;br/&gt;
                                &quot;n&quot; : 1,&lt;br/&gt;
                                &quot;millis&quot; : 71,&lt;br/&gt;
                                &quot;nYields&quot; : 0,&lt;br/&gt;
                                &quot;nChunkSkips&quot; : 0,&lt;br/&gt;
                                &quot;isMultiKey&quot; : false,&lt;br/&gt;
                                &quot;indexOnly&quot; : false,&lt;br/&gt;
                                &quot;indexBounds&quot; : {&lt;/p&gt;

&lt;p&gt;                                }&lt;br/&gt;
                        }&lt;br/&gt;
                ]&lt;br/&gt;
        },&lt;br/&gt;
        &quot;n&quot; : 3,&lt;br/&gt;
        &quot;nChunkSkips&quot; : 6974952,&lt;br/&gt;
        &quot;nYields&quot; : 224,&lt;br/&gt;
        &quot;nscanned&quot; : 7118263,&lt;br/&gt;
        &quot;nscannedObjects&quot; : 7118263,&lt;br/&gt;
        &quot;millisTotal&quot; : 25225,&lt;br/&gt;
        &quot;millisAvg&quot; : 6306,&lt;br/&gt;
        &quot;numQueries&quot; : 4,&lt;br/&gt;
        &quot;numShards&quot; : 4&lt;br/&gt;
}&lt;/p&gt;</comment>
                            <comment id="32347" author="mconigliaro" created="Thu, 12 May 2011 19:00:39 +0000"  >&lt;p&gt;OK, I just reproduced this. It was caused by this script I&apos;m currently using to delete old data:&lt;/p&gt;

&lt;p&gt;&amp;lt;code&amp;gt;&lt;br/&gt;
allowed_events = &lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;quot;click&amp;quot;, &amp;quot;clickThrough&amp;quot;, &amp;quot;conversion&amp;quot;, &amp;quot;emailConversion&amp;quot;, &amp;quot;purchaseConversion&amp;quot;&amp;#93;&lt;/span&gt;&lt;br/&gt;
limit=500;&lt;/p&gt;

&lt;p&gt;print(&quot;total: &quot; + db.userEventsJournal.count())&lt;br/&gt;
i=0;&lt;br/&gt;
db.userEventsJournal.find({ &quot;eventId&quot; : &lt;/p&gt;
{ $nin: allowed_events }
&lt;p&gt; }).limit(limit).forEach(function(o) { /* print(o.eventId); */ db.userEventsJournal.remove({_id: o._id}); i += 1; })&lt;br/&gt;
print(&quot;deleted: &quot; + i);&lt;br/&gt;
&amp;lt;/code&amp;gt;&lt;/p&gt;

&lt;p&gt;When I run it via mongos, it absolutely hammers one of the masters in my replica set (mongodb04). CPU goes to 100% and if it stays pegged long enough, a failover occurs. Now here&apos;s the really interesting thing. When I run this script directly on the master instance, everything works fine. I can even remove 50000+ records at a time with no issues at all. Here&apos;s some explain() output:&lt;/p&gt;

&lt;p&gt;db.userEventsJournal.find({ &quot;eventId&quot; : &lt;/p&gt;
{ $nin: allowed_events }
&lt;p&gt; }).limit(limit).explain()                                                                                                    &lt;br/&gt;
{&lt;br/&gt;
	&quot;cursor&quot; : &quot;BasicCursor&quot;,&lt;br/&gt;
	&quot;nscanned&quot; : 592,&lt;br/&gt;
	&quot;nscannedObjects&quot; : 592,&lt;br/&gt;
	&quot;n&quot; : 500,&lt;br/&gt;
	&quot;millis&quot; : 1,&lt;br/&gt;
	&quot;nYields&quot; : 0,&lt;br/&gt;
	&quot;nChunkSkips&quot; : 0,&lt;br/&gt;
	&quot;isMultiKey&quot; : false,&lt;br/&gt;
	&quot;indexOnly&quot; : false,&lt;br/&gt;
	&quot;indexBounds&quot; : {&lt;/p&gt;

&lt;p&gt;	}&lt;br/&gt;
} &lt;/p&gt;

&lt;p&gt;So it only scans 592 before it finds the 500 I&apos;m interested in. Great! But when I run it via the mongos, I see messages like this on the master:&lt;/p&gt;

&lt;p&gt;Thu May 12 18:15:09 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn22079&amp;#93;&lt;/span&gt; query example.userEventsJournal ntoreturn:5000 reslen:923122 nscanned:7414056 { eventId: &lt;/p&gt;
{ $nin: [ &quot;click&quot;, &quot;clickThrough&quot;, &quot;conversion&quot;, &quot;emailConversion&quot;, &quot;purchaseConversion&quot; ] }
&lt;p&gt; }  nreturned:5000 18539ms&lt;/p&gt;

&lt;p&gt;Notice that via mongos, it&apos;s scanning 7414056 records before it finds 5000 that I&apos;m interested in. I&apos;m sure this is what&apos;s causing the master to become unresponsive, but I don&apos;t understand why the behavior is different when I run the query through a mongos...&lt;/p&gt;</comment>
                            <comment id="32326" author="kristina" created="Thu, 12 May 2011 18:08:20 +0000"  >&lt;p&gt;There seems to be more going on than just the stepdown bug, though&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="15415">SERVER-2882</issuekey>
        </issuelink>
                            </outwardlinks>
                                                        </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>8.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Thu, 12 May 2011 18:08:20 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        11 years, 36 weeks ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ian@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            11 years, 36 weeks ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10000" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Old_Backport</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10000"><![CDATA[No]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10020"><![CDATA[Linux]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>greg_10gen</customfieldvalue>
            <customfieldvalue>ian@mongodb.com</customfieldvalue>
            <customfieldvalue>kristina</customfieldvalue>
            <customfieldvalue>mconigliaro</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrozx3:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrifef:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>21147</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|ht08zj:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>