<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:28:29 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-12413] Assertion on config servers</title>
                <link>https://jira.mongodb.org/browse/SERVER-12413</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Mon Jan 20 16:35:48.110 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2699&amp;#93;&lt;/span&gt; update config.mongos query: &lt;/p&gt;
{ _id: &quot;render-mu04.colo:27017&quot; }
&lt;p&gt; update: { $set: &lt;/p&gt;
{ ping: new Date(1390253748104), up: 111620, waiting: true, mongoVersion: &quot;2.4.8&quot; }
&lt;p&gt; } idhack:1 fastmod:1 keyUpdates:0 exception: assertion src/mongo/db/pdfile.cpp:1816 locks(micros) w:17346 8ms&lt;br/&gt;
Mon Jan 20 16:35:48.149 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2707&amp;#93;&lt;/span&gt; end connection 10.84.150.185:53977 (20 connections now open)&lt;br/&gt;
Mon Jan 20 16:35:48.286 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2709&amp;#93;&lt;/span&gt;  local.oplog.$main Assertion failure !loc.isNull() src/mongo/db/pdfile.cpp 1816&lt;br/&gt;
0xde05e1 0xda15bd 0xab95c1 0xa72524 0xa6b229 0xa8e021 0xa914e5 0xa93847 0x9f6b78 0x9fc0f8 0x6e83a8 0xdccbae 0x3ef9407851 0x3ef90e894d&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo15printStackTraceERSo+0x21) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xde05e1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo12verifyFailedEPKcS1_j+0xfd) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xda15bd&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo11DataFileMgr17fast_oplog_insertEPNS_16NamespaceDetailsEPKci+0x511) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xab95c1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod() &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa72524&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo5logOpEPKcS1_RKNS_7BSONObjEPS2_Pbb+0x49) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa6b229&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod() &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa8e021&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo14_updateObjectsEbPKcRKNS_7BSONObjES4_bbbRNS_7OpDebugEPNS_11RemoveSaverEbRKNS_24QueryPlanSelectionPolicyEb+0x2d35) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa914e5&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo13updateObjectsEPKcRKNS_7BSONObjES4_bbbRNS_7OpDebugEbRKNS_24QueryPlanSelectionPolicyE+0xb7) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa93847&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo14receivedUpdateERNS_7MessageERNS_5CurOpE+0x4d8) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x9f6b78&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo16assembleResponseERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xac8) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x9fc0f8&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0x98) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x6e83a8&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x42e) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xdccbae&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib64/libpthread.so.0() &lt;span class=&quot;error&quot;&gt;&amp;#91;0x3ef9407851&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib64/libc.so.6(clone+0x6d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x3ef90e894d&amp;#93;&lt;/span&gt;&lt;br/&gt;
Mon Jan 20 16:35:48.294 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2709&amp;#93;&lt;/span&gt; update config.mongos query: &lt;/p&gt;
{ _id: &quot;render-mu02.colo:27017&quot; }
&lt;p&gt; update: { $set: &lt;/p&gt;
{ ping: new Date(1390253748288), up: 111618, waiting: false, mongoVersion: &quot;2.4.8&quot; }
&lt;p&gt; } idhack:1 fastmod:1 keyUpdates:0 exception: assertion src/mongo/db/pdfile.cpp:1816 locks(micros) w:17495 8ms&lt;br/&gt;
Mon Jan 20 16:35:48.296 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2709&amp;#93;&lt;/span&gt;  local.oplog.$main Assertion failure !loc.isNull() src/mongo/db/pdfile.cpp 1816&lt;br/&gt;
0xde05e1 0xda15bd 0xab95c1 0xa72524 0xa6b229 0xa8e021 0xa914e5 0xa93847 0x9f6b78 0x9fc0f8 0x6e83a8 0xdccbae 0x3ef9407851 0x3ef90e894d&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo15printStackTraceERSo+0x21) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xde05e1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo12verifyFailedEPKcS1_j+0xfd) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xda15bd&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo11DataFileMgr17fast_oplog_insertEPNS_16NamespaceDetailsEPKci+0x511) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xab95c1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod() &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa72524&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo5logOpEPKcS1_RKNS_7BSONObjEPS2_Pbb+0x49) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa6b229&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod() &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa8e021&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo14_updateObjectsEbPKcRKNS_7BSONObjES4_bbbRNS_7OpDebugEPNS_11RemoveSaverEbRKNS_24QueryPlanSelectionPolicyEb+0x2d35) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa914e5&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo13updateObjectsEPKcRKNS_7BSONObjES4_bbbRNS_7OpDebugEbRKNS_24QueryPlanSelectionPolicyE+0xb7) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa93847&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo14receivedUpdateERNS_7MessageERNS_5CurOpE+0x4d8) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x9f6b78&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo16assembleResponseERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xac8) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x9fc0f8&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0x98) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x6e83a8&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x42e) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xdccbae&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib64/libpthread.so.0() &lt;span class=&quot;error&quot;&gt;&amp;#91;0x3ef9407851&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib64/libc.so.6(clone+0x6d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x3ef90e894d&amp;#93;&lt;/span&gt;&lt;br/&gt;
Mon Jan 20 16:35:48.305 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2709&amp;#93;&lt;/span&gt; update config.mongos query: &lt;/p&gt;
{ _id: &quot;render-mu02.colo:27017&quot; }
&lt;p&gt; update: { $set: &lt;/p&gt;
{ ping: new Date(1390253748298), up: 111618, waiting: true, mongoVersion: &quot;2.4.8&quot; }
&lt;p&gt; } idhack:1 fastmod:1 keyUpdates:0 exception: assertion src/mongo/db/pdfile.cpp:1816 locks(micros) w:17230 8ms&lt;br/&gt;
Mon Jan 20 16:35:49.004 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2709&amp;#93;&lt;/span&gt;  local.oplog.$main Assertion failure !loc.isNull() src/mongo/db/pdfile.cpp 1816&lt;br/&gt;
0xde05e1 0xda15bd 0xab95c1 0xa72524 0xa6b229 0xa90f18 0xa93847 0x9f6b78 0x9fc0f8 0x6e83a8 0xdccbae 0x3ef9407851 0x3ef90e894d&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo15printStackTraceERSo+0x21) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xde05e1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo12verifyFailedEPKcS1_j+0xfd) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xda15bd&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo11DataFileMgr17fast_oplog_insertEPNS_16NamespaceDetailsEPKci+0x511) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xab95c1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod() &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa72524&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo5logOpEPKcS1_RKNS_7BSONObjEPS2_Pbb+0x49) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa6b229&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo14_updateObjectsEbPKcRKNS_7BSONObjES4_bbbRNS_7OpDebugEPNS_11RemoveSaverEbRKNS_24QueryPlanSelectionPolicyEb+0x2768) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa90f18&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo13updateObjectsEPKcRKNS_7BSONObjES4_bbbRNS_7OpDebugEbRKNS_24QueryPlanSelectionPolicyE+0xb7) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa93847&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo14receivedUpdateERNS_7MessageERNS_5CurOpE+0x4d8) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x9f6b78&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo16assembleResponseERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xac8) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x9fc0f8&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0x98) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x6e83a8&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x42e) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xdccbae&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib64/libpthread.so.0() &lt;span class=&quot;error&quot;&gt;&amp;#91;0x3ef9407851&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /lib64/libc.so.6(clone+0x6d) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x3ef90e894d&amp;#93;&lt;/span&gt;&lt;br/&gt;
Mon Jan 20 16:35:49.012 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2709&amp;#93;&lt;/span&gt; update config.lockpings query: &lt;/p&gt;
{ _id: &quot;render-mu02.colo:27017:1390142130:1804289383&quot; }
&lt;p&gt; update: { $set: &lt;/p&gt;
{ ping: new Date(1390253748931) }
&lt;p&gt; } nscanned:1 keyUpdates:1 exception: assertion src/mongo/db/pdfile.cpp:1816 locks(micros) w:16427 8ms&lt;br/&gt;
Mon Jan 20 16:35:49.074 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn2709&amp;#93;&lt;/span&gt; end connection 10.84.150.182:40943 (19 connections now open)&lt;br/&gt;
Mon Jan 20 16:35:50.544 &lt;span class=&quot;error&quot;&gt;&amp;#91;replmaster&amp;#93;&lt;/span&gt;  local.oplog.$main Assertion failure !loc.isNull() src/mongo/db/pdfile.cpp 1816&lt;br/&gt;
0xde05e1 0xda15bd 0xab95c1 0xa72524 0xa6b872 0xb8d045 0xe28e69 0x3ef9407851 0x3ef90e894d&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo15printStackTraceERSo+0x21) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xde05e1&amp;#93;&lt;/span&gt;&lt;/p&gt;
</description>
                <environment>centos, vmware, nexgen</environment>
        <key id="106833">SERVER-12413</key>
            <summary>Assertion on config servers</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="2" iconUrl="https://jira.mongodb.org/images/icons/priorities/critical.svg">Critical - P2</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="bruce.lucas@mongodb.com">Bruce Lucas</assignee>
                                    <reporter username="ilasic">igor lasic</reporter>
                        <labels>
                    </labels>
                <created>Mon, 20 Jan 2014 21:41:58 +0000</created>
                <updated>Tue, 29 Sep 2015 17:43:01 +0000</updated>
                            <resolved>Tue, 29 Sep 2015 17:43:01 +0000</resolved>
                                    <version>2.4.8</version>
                                                    <component>Replication</component>
                                        <votes>0</votes>
                                    <watches>4</watches>
                                                                                                                <comments>
                            <comment id="485268" author="bruce.lucas@10gen.com" created="Tue, 21 Jan 2014 22:25:58 +0000"  >&lt;p&gt;Hi Igor,&lt;/p&gt;

&lt;p&gt;Glad to hear things are working now. Very happy to have been of assistance.&lt;/p&gt;

&lt;p&gt;Bruce&lt;/p&gt;</comment>
                            <comment id="485034" author="ilasic" created="Tue, 21 Jan 2014 18:25:04 +0000"  >&lt;p&gt;Copied the survirving configuration around. Restarted.&lt;/p&gt;

&lt;p&gt;So far so good. &lt;/p&gt;

&lt;p&gt;Closing.&lt;/p&gt;

&lt;p&gt;Thank you for your help.&lt;/p&gt;</comment>
                            <comment id="485017" author="bruce.lucas@10gen.com" created="Tue, 21 Jan 2014 18:08:51 +0000"  >&lt;p&gt;Yes, just stop that config server (to make sure the database files are static), and copy that data, and replicate it to the other two config servers.&lt;/p&gt;

&lt;p&gt;Bruce&lt;/p&gt;</comment>
                            <comment id="484996" author="ilasic" created="Tue, 21 Jan 2014 17:58:29 +0000"  >&lt;p&gt;Definitely San related&lt;/p&gt;

&lt;p&gt;I am following config server restore instructions&lt;br/&gt;
One config server out of 3 survived w/o failures&lt;/p&gt;

&lt;p&gt;Should i copy the data of the surviving one around or is there a different&lt;br/&gt;
recommended procedure?&lt;/p&gt;
</comment>
                            <comment id="484968" author="bruce.lucas@10gen.com" created="Tue, 21 Jan 2014 17:36:15 +0000"  >&lt;p&gt;Hi Igor,&lt;/p&gt;

&lt;p&gt;Thanks for uploading the log. It looks like there is corruption in one of the files of the local database on the config server. I think it&apos;s reasonable to assume that it was caused by a storage error related to the SAN outage; what was the timing of that? The first signs of trouble in the log is the following entry at 12:50:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Sun Jan 19 12:50:30.045 [conn40] couldn&apos;t make room for new record (len: 184) in capped ns local.oplog.$main&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;  Extent 0 (capExtent)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    magic: 41424344 extent-&amp;gt;ns: local.oplog.$main&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    fr: null lr: 0:ce94b8 extent-&amp;gt;len: 5242880&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; local.oplog.$main Assertion failure len * 5 &amp;gt; lastExtentSize src/mongo/db/namespace_details.cpp 457&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;However, because the local.oplog.$main collection is a circular capped collection, it is possible that the corruption occurred some time earlier and was only seen by mongod when the collection wrapped around to the corrupted region.&lt;/p&gt;

&lt;p&gt;To recover from this you can reinitialize that config server, after ensuring that the storage is working normally, following the same procedure as for &lt;a href=&quot;http://docs.mongodb.org/manual/tutorial/replace-config-server/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;replacing a config server&lt;/a&gt;. Before doing that you should be sure do run any relevant hardware diagnostics and &lt;tt&gt;fsck&lt;/tt&gt; the filesystem.&lt;/p&gt;

&lt;p&gt;If you would like us to investigate and look for more definitive evidence that this corruption was caused by the SAN outage, before recovering that config server please take the following steps:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;Upload all currently available kernel logs (dmesg and syslog) from the affected node, including whatever older rotated logs might be available; these might have evidence of the specific problem associated with the corruption. For reference here are some sample commands for collecting those logs:
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;dmesg &amp;gt;/tmp/dmesg&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;tar czf /tmp/system-logs.tgz /var/log/{messages*,syslog*,dmesg*} /tmp/dmesg&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;&lt;/li&gt;
	&lt;li&gt;Upload the local.* database files from the affected node. You can just shut down that config server and then use e.g.  &lt;tt&gt;tar czf /tmp/local.tgz local.*&lt;/tt&gt; in the database directory for that mongod to collect the appropriate files.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;If the resulting two files are less than 150MB and you are comfortable with them being publicly visible you can attach them to this ticket; if they are too large or you would like to keep them private we can provide a secure private location for you to upload them.&lt;/p&gt;

&lt;p&gt;Thanks,&lt;br/&gt;
Bruce&lt;/p&gt;</comment>
                            <comment id="484931" author="ilasic" created="Tue, 21 Jan 2014 16:35:15 +0000"  >&lt;p&gt;one of the config servers log. &lt;/p&gt;


&lt;p&gt;Errors start around Jan 19 11:00&lt;/p&gt;

&lt;p&gt;First error below.&lt;/p&gt;

&lt;p&gt;Sun Jan 19 11:17:21.750 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn60&amp;#93;&lt;/span&gt; end connection 10.84.150.52:43273 (27 connections now open)&lt;br/&gt;
Sun Jan 19 11:17:35.214 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn59&amp;#93;&lt;/span&gt; end connection 10.84.150.51:50992 (26 connections now open)&lt;br/&gt;
Sun Jan 19 11:17:47.547 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn55&amp;#93;&lt;/span&gt; end connection 10.84.150.103:48016 (25 connections now open)&lt;br/&gt;
Sun Jan 19 11:17:48.612 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn61&amp;#93;&lt;/span&gt; end connection 10.84.150.53:49020 (24 connections now open)&lt;br/&gt;
Sun Jan 19 11:17:59.122 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn62&amp;#93;&lt;/span&gt; end connection 10.84.150.54:42804 (23 connections now open)&lt;br/&gt;
Sun Jan 19 11:29:35.317 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn63&amp;#93;&lt;/span&gt; end connection 10.84.150.153:54228 (22 connections now open)&lt;br/&gt;
Sun Jan 19 11:29:35.318 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 10.84.150.153:54943 #64 (23 connections now open)&lt;br/&gt;
Sun Jan 19 11:29:52.674 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn28&amp;#93;&lt;/span&gt; update config.mongos query: &lt;/p&gt;
{ _id: &quot;web01.healthmetrics.org:27017&quot; }
&lt;p&gt; update: { $set: &lt;/p&gt;
{ ping: new Date(1390148992191), up: 6901, waiting: false, mongoVersion: &quot;2.4.8&quot; }
&lt;p&gt; } idhack:1 nupdated:1 fastmod:1 keyUpdates:0 locks(micros) w:958524 479ms&lt;br/&gt;
Sun Jan 19 11:56:37.642 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn64&amp;#93;&lt;/span&gt; end connection 10.84.150.153:54943 (22 connections now open)&lt;br/&gt;
Sun Jan 19 11:56:37.643 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 10.84.150.153:55656 #65 (23 connections now open)&lt;br/&gt;
Sun Jan 19 12:23:40.026 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn65&amp;#93;&lt;/span&gt; end connection 10.84.150.153:55656 (22 connections now open)&lt;br/&gt;
Sun Jan 19 12:23:40.028 &lt;span class=&quot;error&quot;&gt;&amp;#91;initandlisten&amp;#93;&lt;/span&gt; connection accepted from 10.84.150.153:56374 #66 (23 connections now open)&lt;br/&gt;
Sun Jan 19 12:50:30.045 &lt;span class=&quot;error&quot;&gt;&amp;#91;conn40&amp;#93;&lt;/span&gt; couldn&apos;t make room for new record (len: 184) in capped ns local.oplog.$main&lt;br/&gt;
  Extent 0 (capExtent)&lt;br/&gt;
    magic: 41424344 extent-&amp;gt;ns: local.oplog.$main&lt;br/&gt;
    fr: null lr: 0:ce94b8 extent-&amp;gt;len: 5242880&lt;br/&gt;
 local.oplog.$main Assertion failure len * 5 &amp;gt; lastExtentSize src/mongo/db/namespace_details.cpp 457&lt;br/&gt;
0xde05e1 0xda15bd 0xa5e4d8 0x81acaa 0xa5fe39 0xa5fe7c 0xab929c 0xa72524 0xa6b229 0xa8e021 0xa914e5 0xa93847 0x9f6b78 0x9fc0f8 0x6e83a8 0xdccbae 0x3ef9407851 0x3ef90e894d&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo15printStackTraceERSo+0x21) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xde05e1&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo12verifyFailedEPKcS1_j+0xfd) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xda15bd&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZNK5mongo16NamespaceDetails13maybeComplainEPKci+0x928) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa5e4d8&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo16NamespaceDetails11cappedAllocEPKci+0xa8a) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x81acaa&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo16NamespaceDetails6_allocEPKci+0x29) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa5fe39&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo16NamespaceDetails5allocEPKci+0x3c) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa5fe7c&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo11DataFileMgr17fast_oplog_insertEPNS_16NamespaceDetailsEPKci+0x1ec) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xab929c&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod() &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa72524&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo5logOpEPKcS1_RKNS_7BSONObjEPS2_Pbb+0x49) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa6b229&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod() &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa8e021&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo14_updateObjectsEbPKcRKNS_7BSONObjES4_bbbRNS_7OpDebugEPNS_11RemoveSaverEbRKNS_24QueryPlanSelectionPolicyEb+0x2d35) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa914e5&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo13updateObjectsEPKcRKNS_7BSONObjES4_bbbRNS_7OpDebugEbRKNS_24QueryPlanSelectionPolicyE+0xb7) &lt;span class=&quot;error&quot;&gt;&amp;#91;0xa93847&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo14receivedUpdateERNS_7MessageERNS_5CurOpE+0x4d8) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x9f6b78&amp;#93;&lt;/span&gt;&lt;br/&gt;
 /usr/bin/mongod(_ZN5mongo16assembleResponseERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xac8) &lt;span class=&quot;error&quot;&gt;&amp;#91;0x9fc0f8&amp;#93;&lt;/span&gt;&lt;/p&gt;
</comment>
                            <comment id="484816" author="dan@10gen.com" created="Tue, 21 Jan 2014 13:40:09 +0000"  >&lt;p&gt;Can you upload the log files from the config server which is showing the exception since just before the time of the san outage?&lt;/p&gt;</comment>
                            <comment id="484576" author="ilasic" created="Mon, 20 Jan 2014 21:59:41 +0000"  >&lt;p&gt;previous log was from config servers. This is what mongos say:&lt;/p&gt;

&lt;p&gt;    Mon Jan 20 16:57:29.728 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c02:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
    Mon Jan 20 16:57:29.733 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c03:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
    Mon Jan 20 16:57:54.128 &lt;span class=&quot;error&quot;&gt;&amp;#91;LockPinger&amp;#93;&lt;/span&gt; scoped connection to mongo-c01:27019,mongo-c02:27019,mongo-c03:27019 not being returned to the pool&lt;br/&gt;
    Mon Jan 20 16:57:54.128 &lt;span class=&quot;error&quot;&gt;&amp;#91;LockPinger&amp;#93;&lt;/span&gt; warning: distributed lock pinger &apos;mongo-c01:27019,mongo-c02:27019,mongo-c03:27019/render-mu05.colo:27017:1390142128:1804289383&apos; detected an exception while pinging. :: caused by :: update not consistent  ns: config.lockpings query: &lt;/p&gt;
{ _id: &quot;render-mu05.colo:27017:1390142128:1804289383&quot; }
&lt;p&gt; update: { $set: &lt;/p&gt;
{ ping: new Date(1390255073928) }
&lt;p&gt; } gle1: &lt;/p&gt;
{ err: &quot;!loc.isNull()&quot;, n: 0, connectionId: 101, waited: 30, ok: 1.0 }
&lt;p&gt; gle2: &lt;/p&gt;
{ updatedExisting: true, n: 1, lastOp: Timestamp 1390255074000|1, connectionId: 100, waited: 29, err: null, ok: 1.0 }
&lt;p&gt;    Mon Jan 20 16:57:59.778 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c01:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
    Mon Jan 20 16:57:59.779 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c02:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
    Mon Jan 20 16:57:59.781 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c03:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
render-mu01.colo:&lt;br/&gt;
    Mon Jan 20 16:57:07.562 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c01:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
    Mon Jan 20 16:57:07.563 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c02:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
    Mon Jan 20 16:57:07.564 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c03:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
    Mon Jan 20 16:57:33.205 &lt;span class=&quot;error&quot;&gt;&amp;#91;LockPinger&amp;#93;&lt;/span&gt; scoped connection to mongo-c01:27019,mongo-c02:27019,mongo-c03:27019 not being returned to the pool&lt;br/&gt;
    Mon Jan 20 16:57:33.205 &lt;span class=&quot;error&quot;&gt;&amp;#91;LockPinger&amp;#93;&lt;/span&gt; warning: distributed lock pinger &apos;mongo-c01:27019,mongo-c02:27019,mongo-c03:27019/render-mu01.colo:27017:1390145207:1804289383&apos; detected an exception while pinging. :: caused by :: update not consistent  ns: config.lockpings query: &lt;/p&gt;
{ _id: &quot;render-mu01.colo:27017:1390145207:1804289383&quot; }
&lt;p&gt; update: { $set: &lt;/p&gt;
{ ping: new Date(1390255053085) }
&lt;p&gt; } gle1: &lt;/p&gt;
{ err: &quot;!loc.isNull()&quot;, n: 0, connectionId: 81, waited: 28, ok: 1.0 }
&lt;p&gt; gle2: &lt;/p&gt;
{ updatedExisting: true, n: 1, lastOp: Timestamp 1390255053000|1, connectionId: 80, waited: 2, err: null, ok: 1.0 }
&lt;p&gt;    Mon Jan 20 16:57:37.616 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c01:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
    Mon Jan 20 16:57:37.617 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c02:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
    Mon Jan 20 16:57:37.618 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c03:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
    Mon Jan 20 16:58:03.351 &lt;span class=&quot;error&quot;&gt;&amp;#91;LockPinger&amp;#93;&lt;/span&gt; scoped connection to mongo-c01:27019,mongo-c02:27019,mongo-c03:27019 not being returned to the pool&lt;br/&gt;
    Mon Jan 20 16:58:03.351 &lt;span class=&quot;error&quot;&gt;&amp;#91;LockPinger&amp;#93;&lt;/span&gt; warning: distributed lock pinger &apos;mongo-c01:27019,mongo-c02:27019,mongo-c03:27019/render-mu01.colo:27017:1390145207:1804289383&apos; detected an exception while pinging. :: caused by :: update not consistent  ns: config.lockpings query: &lt;/p&gt;
{ _id: &quot;render-mu01.colo:27017:1390145207:1804289383&quot; }
&lt;p&gt; update: { $set: &lt;/p&gt;
{ ping: new Date(1390255083205) }
&lt;p&gt; } gle1: &lt;/p&gt;
{ err: &quot;!loc.isNull()&quot;, n: 0, connectionId: 102, waited: 33, ok: 1.0 }
&lt;p&gt; gle2: &lt;/p&gt;
{ updatedExisting: true, n: 1, lastOp: Timestamp 1390255083000|1, connectionId: 101, waited: 27, err: null, ok: 1.0 }
&lt;p&gt;render-mu04.colo:&lt;br/&gt;
    Mon Jan 20 16:57:21.970 &lt;span class=&quot;error&quot;&gt;&amp;#91;LockPinger&amp;#93;&lt;/span&gt; scoped connection to mongo-c01:27019,mongo-c02:27019,mongo-c03:27019 not being returned to the pool&lt;br/&gt;
    Mon Jan 20 16:57:21.970 &lt;span class=&quot;error&quot;&gt;&amp;#91;LockPinger&amp;#93;&lt;/span&gt; warning: distributed lock pinger &apos;mongo-c01:27019,mongo-c02:27019,mongo-c03:27019/render-mu04.colo:27017:1390142128:1804289383&apos; detected an exception while pinging. :: caused by :: update not consistent  ns: config.lockpings query: &lt;/p&gt;
{ _id: &quot;render-mu04.colo:27017:1390142128:1804289383&quot; }
&lt;p&gt; update: { $set: &lt;/p&gt;
{ ping: new Date(1390255041851) }
&lt;p&gt; } gle1: &lt;/p&gt;
{ err: &quot;!loc.isNull()&quot;, n: 0, connectionId: 79, waited: 16, ok: 1.0 }
&lt;p&gt; gle2: &lt;/p&gt;
{ updatedExisting: true, n: 1, lastOp: Timestamp 1390255041000|1, connectionId: 78, waited: 16, err: null, ok: 1.0 }
&lt;p&gt;    Mon Jan 20 16:57:26.336 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c01:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
    Mon Jan 20 16:57:26.338 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c02:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
    Mon Jan 20 16:57:26.339 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c03:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
    Mon Jan 20 16:57:52.169 &lt;span class=&quot;error&quot;&gt;&amp;#91;LockPinger&amp;#93;&lt;/span&gt; scoped connection to mongo-c01:27019,mongo-c02:27019,mongo-c03:27019 not being returned to the pool&lt;br/&gt;
    Mon Jan 20 16:57:52.169 &lt;span class=&quot;error&quot;&gt;&amp;#91;LockPinger&amp;#93;&lt;/span&gt; warning: distributed lock pinger &apos;mongo-c01:27019,mongo-c02:27019,mongo-c03:27019/render-mu04.colo:27017:1390142128:1804289383&apos; detected an exception while pinging. :: caused by :: update not consistent  ns: config.lockpings query: &lt;/p&gt;
{ _id: &quot;render-mu04.colo:27017:1390142128:1804289383&quot; }
&lt;p&gt; update: { $set: &lt;/p&gt;
{ ping: new Date(1390255071970) }
&lt;p&gt; } gle1: &lt;/p&gt;
{ err: &quot;!loc.isNull()&quot;, n: 0, connectionId: 99, waited: 34, ok: 1.0 }
&lt;p&gt; gle2: &lt;/p&gt;
{ updatedExisting: true, n: 1, lastOp: Timestamp 1390255072000|1, connectionId: 98, waited: 25, err: null, ok: 1.0 }
&lt;p&gt;    Mon Jan 20 16:57:56.392 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c01:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
    Mon Jan 20 16:57:56.393 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c02:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
    Mon Jan 20 16:57:56.394 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c03:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
render-mu02.colo:&lt;br/&gt;
    Mon Jan 20 16:57:24.578 &lt;span class=&quot;error&quot;&gt;&amp;#91;LockPinger&amp;#93;&lt;/span&gt; scoped connection to mongo-c01:27019,mongo-c02:27019,mongo-c03:27019 not being returned to the pool&lt;br/&gt;
    Mon Jan 20 16:57:24.578 &lt;span class=&quot;error&quot;&gt;&amp;#91;LockPinger&amp;#93;&lt;/span&gt; warning: distributed lock pinger &apos;mongo-c01:27019,mongo-c02:27019,mongo-c03:27019/render-mu02.colo:27017:1390142130:1804289383&apos; detected an exception while pinging. :: caused by :: update not consistent  ns: config.lockpings query: &lt;/p&gt;
{ _id: &quot;render-mu02.colo:27017:1390142130:1804289383&quot; }
&lt;p&gt; update: { $set: &lt;/p&gt;
{ ping: new Date(1390255044423) }
&lt;p&gt; } gle1: &lt;/p&gt;
{ err: &quot;!loc.isNull()&quot;, n: 0, connectionId: 80, waited: 10, ok: 1.0 }
&lt;p&gt; gle2: &lt;/p&gt;
{ updatedExisting: true, n: 1, lastOp: Timestamp 1390255044000|1, connectionId: 79, waited: 19, err: null, ok: 1.0 }
&lt;p&gt;    Mon Jan 20 16:57:26.753 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c01:27019&amp;#93;&lt;/span&gt;&lt;br/&gt;
    Mon Jan 20 16:57:26.754 &lt;span class=&quot;error&quot;&gt;&amp;#91;Balancer&amp;#93;&lt;/span&gt; SyncClusterConnection connecting to &lt;span class=&quot;error&quot;&gt;&amp;#91;mongo-c02:27019&amp;#93;&lt;/span&gt;&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="36473" name="mongod.log.gz" size="21723888" author="ilasic" created="Tue, 21 Jan 2014 16:35:15 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>8.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Tue, 21 Jan 2014 13:40:09 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        10 years, 4 weeks, 1 day ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>ramon.fernandez@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            10 years, 4 weeks, 1 day ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10000" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Old_Backport</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10000"><![CDATA[No]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>bruce.lucas@mongodb.com</customfieldvalue>
            <customfieldvalue>dan@mongodb.com</customfieldvalue>
            <customfieldvalue>ilasic</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrm3k7:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrvi5j:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>97784</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10750" key="com.atlassian.jira.plugin.system.customfieldtypes:textarea">
                        <customfieldname>Steps To Reproduce</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>&lt;p&gt;had major san outages and this started showing.&lt;/p&gt;</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                    <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                    <customfieldvalue><![CDATA[bruce.lucas@mongodb.com]]></customfieldvalue>
    

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrj7br:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>