<!-- 
RSS generated by JIRA (9.7.1#970001-sha1:2222b88b221c4928ef0de3161136cc90c8356a66) at Thu Feb 08 03:13:23 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>MongoDB Jira</title>
    <link>https://jira.mongodb.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>9.7.1</version>
        <build-number>970001</build-number>
        <build-date>13-04-2023</build-date>
    </build-info>


<item>
            <title>[SERVER-7008] socket exception [SEND_ERROR] on Mongo Sharding</title>
                <link>https://jira.mongodb.org/browse/SERVER-7008</link>
                <project id="10000" key="SERVER">Core Server</project>
                    <description>&lt;p&gt;Hi i get sometimes the following error, this blocks the whole System. After i restart the Mongod Service it works again well. This was not the first time i get this error.&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;uncaught exception: getlasterror failed: {&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	&quot;shards&quot; : [&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;		&quot;set01/mongo07.luan.local:10011,mongo08.luan.local:10011&quot;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	],&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	&quot;ok&quot; : 0,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;	&quot;errmsg&quot; : &quot;could not get last error from a shard set01/mongo07.luan.local:10011,mongo08.luan.local:10011 :: caused by :: socket exception [SEND_ERROR] for 172.16.18.7:10011&quot;&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;}&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</description>
                <environment>Debian Squeeze / Mongo 2.2.0</environment>
        <key id="50197">SERVER-7008</key>
            <summary>socket exception [SEND_ERROR] on Mongo Sharding</summary>
                <type id="1" iconUrl="https://jira.mongodb.org/secure/viewavatar?size=xsmall&amp;avatarId=14703&amp;avatarType=issuetype">Bug</type>
                                            <priority id="1" iconUrl="https://jira.mongodb.org/images/icons/priorities/blocker.svg">Blocker - P1</priority>
                        <status id="6" iconUrl="https://jira.mongodb.org/images/icons/statuses/closed.png" description="The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.">Closed</status>
                    <statusCategory id="3" key="done" colorName="success"/>
                                    <resolution id="9">Done</resolution>
                                        <assignee username="randolph@mongodb.com">Randolph Tan</assignee>
                                    <reporter username="e.hofmann@activate.de">Enrico Hofmann</reporter>
                        <labels>
                    </labels>
                <created>Tue, 11 Sep 2012 08:43:05 +0000</created>
                <updated>Thu, 6 Apr 2023 16:24:29 +0000</updated>
                            <resolved>Wed, 19 Mar 2014 02:56:51 +0000</resolved>
                                    <version>2.2.0</version>
                                                    <component>Networking</component>
                    <component>Sharding</component>
                                        <votes>20</votes>
                                    <watches>44</watches>
                                                                                                                <comments>
                            <comment id="674392" author="vishal.katikineni@valuelabs.com" created="Tue, 29 Jul 2014 01:37:48 +0000"  >&lt;p&gt;We have a similar issue too even after setting the parameter releaseConnectionsAfterResponse=true.&lt;/p&gt;</comment>
                            <comment id="523362" author="apiggott@ikanow.com" created="Wed, 26 Mar 2014 02:23:27 +0000"  >&lt;p&gt;Created &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-13352&quot; title=&quot;Socket exception (SEND_ERROR) even after SERVER-9022 applied&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-13352&quot;&gt;&lt;del&gt;SERVER-13352&lt;/del&gt;&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="521543" author="eliot" created="Mon, 24 Mar 2014 13:16:56 +0000"  >&lt;p&gt;Alex - can you open a new ticket (and link here) just to make sure we don&apos;t confuse two issues?&lt;/p&gt;</comment>
                            <comment id="521506" author="apiggott@ikanow.com" created="Mon, 24 Mar 2014 11:30:57 +0000"  >&lt;p&gt;hmm i spoke too soon, i added this fix at the start of feb and from early last week (ie mid march) the same problem started recurring, across more and more nodes (32 nodes running mongos 2.4.9, 20 of them run mongod in 10 shards) over a few days, as before.&lt;/p&gt;

&lt;p&gt;As before, a mass mongos restart fixed everything.&lt;/p&gt;

&lt;p&gt;it looks like you missed a less frequent case of the same sort of problem (before the fix, it happened every 1-2 weeks)?&lt;/p&gt;

&lt;p&gt;Should his issue be reopened, or &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-9022&quot; title=&quot;Enable mongos mode which releases ShardConnections to the pool after read ops&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-9022&quot;&gt;&lt;del&gt;SERVER-9022&lt;/del&gt;&lt;/a&gt;, or should i create a new issue?&lt;/p&gt;</comment>
                            <comment id="505448" author="apiggott@ikanow.com" created="Wed, 26 Feb 2014 02:58:49 +0000"  >&lt;p&gt;In case anyone was wondering, since I applied &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-9022&quot; title=&quot;Enable mongos mode which releases ShardConnections to the pool after read ops&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-9022&quot;&gt;&lt;del&gt;SERVER-9022&lt;/del&gt;&lt;/a&gt; to my servers this problem has completely disappeared and I have seen no other ill effects. Thanks!&lt;/p&gt;</comment>
                            <comment id="488595" author="dan@10gen.com" created="Tue, 28 Jan 2014 05:25:21 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-9022&quot; title=&quot;Enable mongos mode which releases ShardConnections to the pool after read ops&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-9022&quot;&gt;&lt;del&gt;SERVER-9022&lt;/del&gt;&lt;/a&gt; will mitigate the issue under most circumstances, but not completely fix the issue.  If your application conforms to the caveats described in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-9022&quot; title=&quot;Enable mongos mode which releases ShardConnections to the pool after read ops&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-9022&quot;&gt;&lt;del&gt;SERVER-9022&lt;/del&gt;&lt;/a&gt; (most apps should conform), then I encourage you to turn on &lt;tt&gt;releaseConnectionsAfterResponse&lt;/tt&gt; on a single mongos and monitor the difference.  This setting will effectively be on in 2.6 by default (the option will be deprecated in 2.6).&lt;/p&gt;</comment>
                            <comment id="488351" author="apiggott@ikanow.com" created="Mon, 27 Jan 2014 20:28:26 +0000"  >&lt;p&gt;It just happened again, this time running a command from the shell did not fix it (there must just have been a cluster of working servers that the round robin happened to hit when I was trying that).&lt;/p&gt;

&lt;p&gt;I also tried the &quot;db.adminCommand( &lt;/p&gt;
{ flushRouterConfig : 1 }
&lt;p&gt; )&quot; and (as with one of the other commenters) that also definitely did not work&lt;/p&gt;

&lt;p&gt;Restarting the mongos (and then all of the follow on processes connected via a Mongo object) did work, as before.&lt;/p&gt;

&lt;p&gt;It would be really handy if someone from 10gen could confirm whether &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-9022&quot; title=&quot;Enable mongos mode which releases ShardConnections to the pool after read ops&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-9022&quot;&gt;&lt;del&gt;SERVER-9022&lt;/del&gt;&lt;/a&gt; will work given the flushRouterConfig doesn&apos;t? If it is believed to work I will try it out and report back, otherwise I suppose I&apos;ll try restarting at 2am every few days in case it&apos;s a &quot;leak&quot; type problem rather than a &quot;race&quot; type problem.&lt;/p&gt;</comment>
                            <comment id="484902" author="apiggott@ikanow.com" created="Tue, 21 Jan 2014 15:47:05 +0000"  >&lt;p&gt;I would be odd if it was a complete coincidence given how consistent the behaviour was:&lt;/p&gt;
&lt;ul&gt;
	&lt;li&gt;I had a GUI that calls find(XXX).count() via the JAVA driver when I press a button. I pressed the button several times, each time I got an error.&lt;/li&gt;
	&lt;li&gt;I then entered the shell via ssh, ran the count command (which always worked), and then pressed the button again several times and now each time it worked.&lt;/li&gt;
&lt;/ul&gt;


&lt;p&gt;I did this on a few different servers. &lt;/p&gt;

&lt;p&gt;Is it possible that logging into the mongos via the shell (or calling the &quot;use&quot; command, or calling the &quot;count&quot; command) resets the connections?&lt;/p&gt;

&lt;p&gt;Incidentally will &quot;db.adminCommand( &lt;/p&gt;
{ flushRouterConfig : 1 }
&lt;p&gt; )&quot; break any commands that are ongoing when I run it? &lt;/p&gt;

&lt;p&gt;I&apos;d like to run it regularly until we can move to a version that has a &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-9041&quot; title=&quot;proactively detect broken connections detected by the network&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-9041&quot;&gt;&lt;del&gt;SERVER-9041&lt;/del&gt;&lt;/a&gt; (unless that will cause performance issues, eg if I run it every 10 minutes?) and see if that prevents the problem from occurring.&lt;/p&gt;</comment>
                            <comment id="484893" author="renctan" created="Tue, 21 Jan 2014 15:18:48 +0000"  >&lt;p&gt;I believe that it was a coincidence that calling count fixes the problem. This issue would eventually resolve itself once all the bad connections are replaced with new ones. Prior to &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-9041&quot; title=&quot;proactively detect broken connections detected by the network&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-9041&quot;&gt;&lt;del&gt;SERVER-9041&lt;/del&gt;&lt;/a&gt;, mongos will finds out if the connection was bad/closed only after using it and throwing an error.&lt;/p&gt;</comment>
                            <comment id="484861" author="apiggott@ikanow.com" created="Tue, 21 Jan 2014 14:45:27 +0000"  >&lt;p&gt;For what it&apos;s worth - I&apos;ve been getting this quite a lot (on non-trivial &quot;count()&quot; commands only) since I built a 28-node cluster (14 shards x 2 replicas) ... My previous largest cluster was 4 nodes (2 shards x 2 replicas), and only saw this error very intermittently (and not in clusters, just the odd one off from time to time).&lt;/p&gt;

&lt;p&gt;Monday morning every single mongos (on 14 separate application nodes) was in this errored state. Restarting each mongos fixed it on that mongos - but in addition just typing &quot;use &amp;lt;database&amp;gt;&quot;, &quot;db.&amp;lt;collection&amp;gt;.count()&quot; in the local shell also appeared to fix it (for the &amp;lt;database&amp;gt; and  &amp;lt;collection&amp;gt; in question).&lt;/p&gt;

&lt;p&gt;Thought I&apos;d mention these data points in case they help diagnose the problem, and in addition if the simple &quot;count()&quot; on the collection really fixes it (in some cases?!), then that might be a workaround for some people pending a full fix.&lt;/p&gt;

&lt;p&gt;Has anyone actually tried &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-9022&quot; title=&quot;Enable mongos mode which releases ShardConnections to the pool after read ops&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-9022&quot;&gt;&lt;del&gt;SERVER-9022&lt;/del&gt;&lt;/a&gt;? The problem only occurs on my operational clusters, I don&apos;t really want mess about with a new mongos mode unless there&apos;s a good reason.&lt;/p&gt;</comment>
                            <comment id="484649" author="jessecai" created="Tue, 21 Jan 2014 01:58:24 +0000"  >&lt;p&gt;Confirmed this issue still exist in 2.4.6, it happening after some of the members were rebooted. it will back to normal after a full restart of mongos.&lt;/p&gt;</comment>
                            <comment id="468418" author="bgsosh@gmail.com" created="Tue, 10 Dec 2013 08:16:11 +0000"  >&lt;p&gt;Getting the same problem with a 2.4.5 cluster.  Started happening after some of the members were rebooted.  Does not seem to be clearing itself, but can&apos;t find anything wrong...&lt;/p&gt;</comment>
                            <comment id="457350" author="alexandrov" created="Sat, 16 Nov 2013 10:18:45 +0000"  >&lt;p&gt;Hi&lt;/p&gt;

&lt;p&gt;The same problem&lt;/p&gt;

&lt;p&gt;MongoS version 2.4.8 starting: pid=14289 port=27017 64-bit host=second-w3.x.xx.xx (--help for usage)&lt;br/&gt;
git version: a350fc38922fbda2cec8d5dd842237b904eafc14&lt;br/&gt;
build sys info: Linux ip-10-2-29-40 2.6.21.7-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49&lt;/p&gt;

&lt;p&gt;Traceback (most recent call last):&lt;br/&gt;
  File &quot;/usr/lib/python2.6/site-packages/bottle.py&quot;, line 764, in _handle&lt;br/&gt;
    return route.call(**args)&lt;br/&gt;
  File &quot;/usr/lib/python2.6/site-packages/bottle.py&quot;, line 1575, in wrapper&lt;br/&gt;
    rv = callback(*a, **ka)&lt;br/&gt;
  File &quot;/var/www/desk/mobile.py&quot;, line 526, in article&lt;br/&gt;
    content = self.store. xx_ru.Name.find_one(&lt;/p&gt;
{&quot;_id&quot;: int(article_id)}
&lt;p&gt;)&lt;br/&gt;
  File &quot;/usr/lib64/python2.6/site-packages/pymongo/collection.py&quot;, line 604, in find_one&lt;br/&gt;
    for result in self.find(spec_or_id, *args, **kwargs).limit(-1):&lt;br/&gt;
  File &quot;/usr/lib64/python2.6/site-packages/pymongo/cursor.py&quot;, line 904, in next&lt;br/&gt;
    if len(self.__data) or self._refresh():&lt;br/&gt;
  File &quot;/usr/lib64/python2.6/site-packages/pymongo/cursor.py&quot;, line 848, in _refresh&lt;br/&gt;
    self.__uuid_subtype))&lt;br/&gt;
  File &quot;/usr/lib64/python2.6/site-packages/pymongo/cursor.py&quot;, line 800, in __send_message&lt;br/&gt;
    self.__uuid_subtype)&lt;br/&gt;
  File &quot;/usr/lib64/python2.6/site-packages/pymongo/helpers.py&quot;, line 100, in _unpack_response&lt;br/&gt;
    error_object&lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;quot;$err&amp;quot;&amp;#93;&lt;/span&gt;)&lt;br/&gt;
OperationFailure: database error: socket exception &lt;span class=&quot;error&quot;&gt;&amp;#91;SEND_ERROR&amp;#93;&lt;/span&gt; for 10.0.xx.xx:27117&lt;/p&gt;

&lt;p&gt;bill goes to the days of the conclusion of the system is needed in productivity &lt;img class=&quot;emoticon&quot; src=&quot;https://jira.mongodb.org/images/icons/emoticons/sad.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/p&gt;</comment>
                            <comment id="446806" author="eyllo" created="Fri, 25 Oct 2013 17:09:30 +0000"  >&lt;p&gt;Hi there, &lt;/p&gt;

&lt;p&gt;We are also hitting this problem using 2.4.7 inside Microsoft Azure and having all of our servers within the same affinity group and region. Any ideas when this will be fixed? or how it can be solved?&lt;/p&gt;</comment>
                            <comment id="423691" author="renctan" created="Thu, 12 Sep 2013 17:57:31 +0000"  >&lt;p&gt;Hi,&lt;/p&gt;

&lt;p&gt;We also made some changes in 2.5 that would clear the bad connections sooner in mongos with &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-9041&quot; title=&quot;proactively detect broken connections detected by the network&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-9041&quot;&gt;&lt;del&gt;SERVER-9041&lt;/del&gt;&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;Thanks!&lt;/p&gt;</comment>
                            <comment id="420522" author="airesp" created="Sun, 8 Sep 2013 22:30:23 +0000"  >&lt;p&gt;Hi!&lt;br/&gt;
Somebody tried to work options in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-9022&quot; title=&quot;Enable mongos mode which releases ShardConnections to the pool after read ops&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-9022&quot;&gt;&lt;del&gt;SERVER-9022&lt;/del&gt;&lt;/a&gt;?&lt;/p&gt;

&lt;p&gt;This bug makes it impossible to develop the project in conjunction with mongodb &lt;img class=&quot;emoticon&quot; src=&quot;https://jira.mongodb.org/images/icons/emoticons/sad.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/p&gt;</comment>
                            <comment id="305038" author="eliot" created="Wed, 3 Apr 2013 13:39:03 +0000"  >&lt;p&gt;James - I think the evidence is inconclusive at best.&lt;/p&gt;

&lt;p&gt;2.2.4 is out, so assuming the caveat&apos;s listed in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-9022&quot; title=&quot;Enable mongos mode which releases ShardConnections to the pool after read ops&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-9022&quot;&gt;&lt;del&gt;SERVER-9022&lt;/del&gt;&lt;/a&gt; do not apply, I would encourage you to try that.&lt;/p&gt;</comment>
                            <comment id="304955" author="jblackburn" created="Wed, 3 Apr 2013 10:38:50 +0000"  >&lt;p&gt;I&apos;d assumed this because the errors often seem to be clustered in the morning:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;list(c.cube.ahl_mongo_events.find({&apos;d.error&apos;: {&apos;$regex&apos;: &apos;.* socket exception .*&apos;}}, fields={&apos;t&apos;:1, &apos;_id&apos;:0}))&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;...&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 13, 7, 56, 33, 957000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 13, 7, 56, 34, 28000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 13, 7, 56, 34, 252000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 13, 8, 21, 10, 997000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 13, 8, 49, 11, 866000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 13, 9, 28, 15, 914000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 13, 10, 3, 10, 779000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 13, 10, 3, 10, 812000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 14, 8, 30, 19, 850000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 14, 8, 30, 19, 987000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 14, 9, 53, 17, 560000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 14, 9, 53, 17, 628000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 14, 9, 53, 17, 666000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 14, 9, 54, 2, 400000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 14, 9, 54, 2, 432000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 14, 10, 30, 51, 442000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 14, 11, 20, 49, 533000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; {u&apos;t&apos;: datetime.datetime(2013, 3, 14, 14, 48, 44, 212000)},&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;...&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Perhaps the assumption is wrong, as the exceptions do happen in the afternoon too.&lt;/p&gt;


&lt;p&gt;Digging into the tracebacks, they nearly always occur in a cursor &apos;refresh&apos;, which I hadn&apos;t noticed before:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;c.cube.ahl_mongo_events.find({&apos;d.error&apos;: {&apos;$regex&apos;: &apos;.* socket exception .*&apos;}}).distinct(&apos;d.traceback&apos;)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;...&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;  File &quot;/users/is/ahlpypi/egg_cache/p/pymongo-2.4.2-py2.6-linux-x86_64.egg/pymongo/cursor.py&quot;, line 814, in next&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    if len(self.__data) or self._refresh():&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;  File &quot;/users/is/ahlpypi/egg_cache/p/pymongo-2.4.2-py2.6-linux-x86_64.egg/pymongo/cursor.py&quot;, line 763, in _refresh&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    self.__uuid_subtype))&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;  File &quot;/users/is/ahlpypi/egg_cache/p/pymongo-2.4.2-py2.6-linux-x86_64.egg/pymongo/cursor.py&quot;, line 720, in __send_message&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    self.__uuid_subtype)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;  File &quot;/users/is/ahlpypi/egg_cache/p/pymongo-2.4.2-py2.6-linux-x86_64.egg/pymongo/helpers.py&quot;, line 105, in _unpack_response&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    error_object[&quot;$err&quot;])&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;OperationFailure: database error: socket exception [SEND_ERROR] for 10.220.54.51:27118&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;...&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;len([x for x in c.cube.ahl_mongo_events.find({&apos;d.error&apos;: {&apos;$regex&apos;: &apos;.* socket exception .*&apos;}}) if &apos;_refresh&apos; in x[&apos;d&apos;][&apos;traceback&apos;]])&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Out[81]: 446&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;len([x for x in c.cube.ahl_mongo_events.find({&apos;d.error&apos;: {&apos;$regex&apos;: &apos;.* socket exception .*&apos;}}) if &apos;_refresh&apos; not in x[&apos;d&apos;][&apos;traceback&apos;]])&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Out[82]: 25&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Those 25 are errors in waiting for command response (_check_command_response in the Python):&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[x[&apos;d&apos;][&apos;error&apos;] for x in c.cube.ahl_mongo_events.find({&apos;d.error&apos;: {&apos;$regex&apos;: &apos;.* socket exception .*&apos;}}) if &apos;_refresh&apos; not in x[&apos;d&apos;][&apos;traceback&apos;]]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Out[92]: &lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;[u&quot;command SON([(&apos;findAndModify&apos;, u&apos;EOD.version_nums&apos;), ....]) failed: exception: socket exception [SEND_ERROR] for 10.220.54.14:27118&quot;,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt; u&apos;command SON([(\&apos;count\&apos;, u\&apos;EOD\&apos;), (\&apos;fields\&apos;, None), ...)]) failed: exception: socket exception [CONNECT_ERROR] for rs1/cn54-ib:27118,cn55-ib:27118&apos;,&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;... ]&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&amp;nbsp;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

</comment>
                            <comment id="304574" author="eliot" created="Tue, 2 Apr 2013 21:59:17 +0000"  >&lt;p&gt;James - why do you think they are coming out of the global pool?&lt;/p&gt;</comment>
                            <comment id="304369" author="jblackburn" created="Tue, 2 Apr 2013 18:45:28 +0000"  >&lt;p&gt;Indeed, I don&apos;t think Server-9022 will help. &lt;/p&gt;

&lt;p&gt;My experience is that idle sockets collect in the global pool. When they are next used, they error. It would be great if sockets in the global pool expired after a fixed timeout. This would prevent the collection of stale sockets and OperationFailure exceptions we see every morning. &lt;/p&gt;</comment>
                            <comment id="304313" author="JIRAUSER1253508" created="Tue, 2 Apr 2013 17:59:37 +0000"  >&lt;p&gt;Thank you we will give it a try as soon as it is available.&lt;/p&gt;

&lt;p&gt;The problem is not so much that it is not returning a connection to the pool. The problem is that it is returning bad/failed connections to the pool. When a socket error occurs on a connection it should be removed from the pool and destroyed. What we are seeing is that bad connections in the pool are being re-used resulting in constant send failures in mongos for that bad connection.&lt;/p&gt;</comment>
                            <comment id="304220" author="eliot" created="Tue, 2 Apr 2013 16:04:50 +0000"  >&lt;p&gt;For 2.2 and 2.4, there is a new option in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-9022&quot; title=&quot;Enable mongos mode which releases ShardConnections to the pool after read ops&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-9022&quot;&gt;&lt;del&gt;SERVER-9022&lt;/del&gt;&lt;/a&gt;.&lt;br/&gt;
When 2.2.4 comes out (this week) would be great if some people could help test it.&lt;br/&gt;
Its off by default for reasons described in that ticket.&lt;/p&gt;</comment>
                            <comment id="304140" author="renctan" created="Tue, 2 Apr 2013 14:39:49 +0000"  >&lt;p&gt;Hi,&lt;/p&gt;

&lt;p&gt;We are still working on this. We are current experimenting on a fix explained in this linked &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-9041&quot; title=&quot;proactively detect broken connections detected by the network&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-9041&quot;&gt;&lt;del&gt;ticket&lt;/del&gt;&lt;/a&gt; which is planned for 2.5.&lt;/p&gt;</comment>
                            <comment id="303841" author="geordios" created="Tue, 2 Apr 2013 04:17:21 +0000"  >&lt;p&gt;Yes it is very quiet and still very quiet. Is there any update on this? &lt;/p&gt;</comment>
                            <comment id="292601" author="eliot" created="Tue, 19 Mar 2013 01:20:07 +0000"  >&lt;p&gt;matt - yes, much too quiet.  There was a lot of internal communication about it, so didn&apos;t realize so little made it out publicly.&lt;/p&gt;</comment>
                            <comment id="292599" author="mbailey@gild.com" created="Tue, 19 Mar 2013 01:18:42 +0000"  >&lt;p&gt;@eliot Excellent, really appreciate the update from 10gen, seemed pretty quiet on this ticket.&lt;/p&gt;</comment>
                            <comment id="292598" author="eliot" created="Tue, 19 Mar 2013 01:17:22 +0000"  >&lt;p&gt;There are 2 other changes in 2.2.4 that might be related as well in some cases: &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-8786&quot; title=&quot;Race condition when setting ShardingConnectionHook on mongod connection pools&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-8786&quot;&gt;&lt;del&gt;SERVER-8786&lt;/del&gt;&lt;/a&gt; and &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-8648&quot; title=&quot;checkStatus in DBClientReplicaSet has no explicit socket timeout&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-8648&quot;&gt;&lt;del&gt;SERVER-8648&lt;/del&gt;&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="292594" author="eliot" created="Tue, 19 Mar 2013 01:13:49 +0000"  >&lt;p&gt;Definitely don&apos;t have to wait for 2.6.&lt;/p&gt;

&lt;p&gt;We&apos;re working with one user on a patch that seems to help a lot in 2.2&lt;/p&gt;

&lt;p&gt;I think the problem is that there are a lot of potential causes, some networking, some system, so its been very hard to get a clear picture of what&apos;s happening.&lt;/p&gt;

&lt;p&gt;Should have a patch for 2.2.4 or 2.2.5 that in most cases should greatly reduce this issue, or at least rule out large buckets of potential issues.&lt;/p&gt;</comment>
                            <comment id="292477" author="openmosix" created="Mon, 18 Mar 2013 22:24:12 +0000"  >&lt;p&gt;We&apos;ve been advocating Mongo for a long time, and this is the first time a bug hit us so badly in production and for such a long time. The answer of disabling the sharding or wait till 2.6 (2 major versions away!) it&apos;s simply a joke. We have built scripts and monitors to detect when this happens and try to restart faulty mongo routers (restart everything feels very Windows 95 btw). We were using Mongo for a high performance cluster, and we&apos;re down basically every other hour.&lt;/p&gt;

&lt;p&gt;We have been early adopters and supporters while a lot of negative criticisms were depicting Mongo as a toy.&lt;br/&gt;
Today we really feel you let us down, and we&apos;ve initiated our plan to ditch Mongo for others data storage. This is not only hurting our business, but also your business partners - like MongoHQ, that they have been great with us. It&apos;s a sad day.&lt;/p&gt;</comment>
                            <comment id="292469" author="noizwaves" created="Mon, 18 Mar 2013 22:16:10 +0000"  >&lt;p&gt;@DisU seconded. We use Mongo because of it&apos;s easy clustering. If this doesn&apos;t work reliably, this is a real concern for us and the service we provide to our customers!&lt;/p&gt;</comment>
                            <comment id="292462" author="disappointed" created="Mon, 18 Mar 2013 22:03:13 +0000"  >&lt;p&gt;I completely agree that this issue needs to be addressed, even as part of 2.2.4. Retrying five times is not an acceptable workaround. We&apos;re considering migrating away from Mongo because of this and similar issues. &lt;/p&gt;</comment>
                            <comment id="292353" author="mbailey@gild.com" created="Mon, 18 Mar 2013 20:20:02 +0000"  >&lt;p&gt;Is there anything we can do to mitigate this issue, or are we really going to have to wait for 2.6 for this to be addressed?&lt;/p&gt;</comment>
                            <comment id="289440" author="jblackburn" created="Thu, 14 Mar 2013 19:53:53 +0000"  >&lt;p&gt;We&apos;ve been trialling Mongo here, and this is the bug that hurts us most.  &lt;/p&gt;

&lt;p&gt;It&apos;s difficult to workaround too - as it&apos;s unclear how many stale connections the mongos has especially in the morning after a night without use.  So we end up retrying 5 times (around &lt;em&gt;every&lt;/em&gt; mongo operation) and sometimes that isn&apos;t enough.&lt;/p&gt;</comment>
                            <comment id="278903" author="jblackburn" created="Fri, 1 Mar 2013 11:09:22 +0000"  >&lt;p&gt;I&apos;ve written a multi-process batch job.  What&apos;s interesting is that when this error occurs against the local mongos, it seems to occur consistently for one PID on the local machine at a time.  Trying to get it to happen reliably is difficult, and it randomly fixes itself.  But for the duration (order of minutes) it appears that the given mongo connection doesn&apos;t work.&lt;/p&gt;</comment>
                            <comment id="257889" author="disappointed" created="Mon, 4 Feb 2013 19:54:38 +0000"  >&lt;p&gt;Any news on this? We&apos;ve been experiencing the same issue in the last few months.&lt;/p&gt;</comment>
                            <comment id="252142" author="JIRAUSER1253508" created="Mon, 28 Jan 2013 20:57:01 +0000"  >&lt;p&gt;I confirmed that on all of the Redhat Linux servers we have the tcp keep-alive time set to 300&lt;/p&gt;</comment>
                            <comment id="252123" author="JIRAUSER1253508" created="Mon, 28 Jan 2013 20:43:20 +0000"  >&lt;p&gt;Yes, we are using the slaveOk read preference. The network connection issues are generally to the replica-set masters since they are on a different network segment.&lt;/p&gt;

&lt;p&gt;It is possible that the network connections to the master are going &quot;stale&quot; since all reads are to the slaves and writes are seldom performed.&lt;/p&gt;</comment>
                            <comment id="252111" author="JIRAUSER1253508" created="Mon, 28 Jan 2013 20:33:57 +0000"  >&lt;p&gt;We are running 5 shards each of which is a replica-set. We use the Ruby and Go-lang drivers so it is not likely to related to the driver in any way.&lt;/p&gt;

&lt;p&gt;To reproduce the problem:&lt;br/&gt;
1. Setup a shard of X replica sets&lt;br/&gt;
2. Run the mongos instance on your local machine&lt;br/&gt;
3. Perform a query from a test program which goes through the local mongos&lt;br/&gt;
4. Break the network connection between the master and mongos&lt;br/&gt;
5. Perform another query which should fail with a connection lost OperationFailure&lt;br/&gt;
6. Re-establish the network connection between mongos and the master&lt;br/&gt;
7. Perform another query which should fail with a connection lost OperationFailure&lt;/p&gt;

&lt;p&gt;Step 7 may have to be repeated several times since the connections between mongos and the master are in a connection pool in mongos so there is no guarantee that the bad connection will be used on the first attempt&lt;/p&gt;

&lt;p&gt;Once mongos is in this state the only way to fix it is to restart the mongos process. It is not necessary to restart the client.&lt;/p&gt;

&lt;p&gt;The longer mongos is left running the greater the number of these bad connections in its connection pool, thereby increasing the likelihood of getting a bad connection when a call is made via mongos.&lt;/p&gt;</comment>
                            <comment id="251938" author="jblackburn" created="Mon, 28 Jan 2013 17:40:44 +0000"  >&lt;p&gt;The shards are replicasets.  We&apos;re using pymongo 2.4.1, the clients are coming in with a read-preference set to primary only.&lt;/p&gt;

&lt;p&gt;One thing which is (very) non-standard here is that we&apos;re LD_PRELOAD&apos;ing of libsdp to run TCP/IP over Infiniband.  It looks like this doesn&apos;t honour the TCP keepalive request &amp;#8211; I see this in the mongos log files:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Mon Jan 28 08:27:24 [conn129] Socket say send() errno:104 Connection reset by peer 10.220.54.17:27118&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Mon Jan 28 08:27:24 [conn129] DBException in process: socket exception [SEND_ERROR] for 10.220.54.17:27118&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;Mon Jan 28 08:27:24 [conn129] ERROR: can&apos;t get TCP_KEEPINTVL: errno:92 Protocol not available&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Other than these periodic OperationFailures, Mongo over infiniband seems to work very well indeed.&lt;/p&gt;</comment>
                            <comment id="251842" author="renctan" created="Mon, 28 Jan 2013 16:15:16 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=reidmorrison&quot; class=&quot;user-hover&quot; rel=&quot;reidmorrison&quot;&gt;reidmorrison&lt;/a&gt; &amp;amp; &lt;a href=&quot;https://jira.mongodb.org/secure/ViewProfile.jspa?name=jblackburn&quot; class=&quot;user-hover&quot; rel=&quot;jblackburn&quot;&gt;jblackburn&lt;/a&gt;: Are the shards replica-sets or stand alone servers? If it&apos;s the former, what is the version of the driver you are using and are you using slaveOk/read preference?&lt;/p&gt;

&lt;p&gt;Thanks!&lt;/p&gt;</comment>
                            <comment id="251737" author="greg_10gen" created="Mon, 28 Jan 2013 14:43:39 +0000"  >&lt;p&gt;Looking more into this.  One potential fix/workaround may be keepalive - if it&apos;s not set to a low value already, does setting keepalive mitigate the problem?&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;http://docs.mongodb.org/manual/faq/diagnostics/#does-tcp-keepalive-time-affect-sharded-clusters-and-replica-sets&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://docs.mongodb.org/manual/faq/diagnostics/#does-tcp-keepalive-time-affect-sharded-clusters-and-replica-sets&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;If anyone has a sample of the logs from mongos during the affected period (from the start of mongos, to include startup parameters), ideally at log level 3 (can be set at runtime), it would be useful to see exactly the timings of what is happening here on a per-conn basis.  If they contain sensitive info, opening a SUPPORT ticket (which is user-private) linked to this ticket is an option as well.&lt;/p&gt;</comment>
                            <comment id="247531" author="jblackburn" created="Wed, 23 Jan 2013 13:06:03 +0000"  >&lt;p&gt;We get a whole load of these, often in the morning. Which suggests the sockets are timing out overnight and not being noticed.  Currently I&apos;m logging all OperationFailures, and these look like:&lt;/p&gt;

&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;  File &quot;/users/is/ahlpypi/egg_cache/p/pymongo-2.4.1_1-py2.6-linux-x86_64.egg/pymongo/collection.py&quot;, line 598, in find_one&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    for result in self.find(spec_or_id, *args, **kwargs).limit(-1):&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;  File &quot;/users/is/ahlpypi/egg_cache/p/pymongo-2.4.1_1-py2.6-linux-x86_64.egg/pymongo/cursor.py&quot;, line 814, in next&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    if len(self.__data) or self._refresh():&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;  File &quot;/users/is/ahlpypi/egg_cache/p/pymongo-2.4.1_1-py2.6-linux-x86_64.egg/pymongo/cursor.py&quot;, line 763, in _refresh&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    self.__uuid_subtype))&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;  File &quot;/users/is/ahlpypi/egg_cache/p/pymongo-2.4.1_1-py2.6-linux-x86_64.egg/pymongo/cursor.py&quot;, line 720, in __send_message&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    self.__uuid_subtype)&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;  File &quot;/users/is/ahlpypi/egg_cache/p/pymongo-2.4.1_1-py2.6-linux-x86_64.egg/pymongo/helpers.py&quot;, line 104, in _unpack_response&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;    error_object[&quot;$err&quot;])&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;OperationFailure: database error: socket exception [SEND_ERROR] for 10.220.54.55:27118&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;

&lt;p&gt;Retrying seems to resolve the problem (I only see them logged once for a given client).  It&apos;s a shame that they&apos;re not represented as AutoReconnect errors, as it means we have to put AutoReconnect and OperationFailure decorators around every single pymongo call.&lt;/p&gt;</comment>
                            <comment id="230978" author="JIRAUSER1253508" created="Fri, 4 Jan 2013 16:24:57 +0000"  >&lt;p&gt;We are definitely seeing a dramatic difference since upgrading production from V2.0.6 to V2.2.2. It only affects the mongos processes in our second (remote) data center that perform writes back to the shard masters in the first data center. Anytime there is a network issue between the 2 data centers all the mongos processes in the remote data center have to be restarted due to the send errors being sent back to the clients.&lt;/p&gt;

&lt;p&gt;We run one mongos process on every web and batch processing server to reduce network latency and to improve redundancy.&lt;/p&gt;</comment>
                            <comment id="230972" author="JIRAUSER1253508" created="Fri, 4 Jan 2013 16:16:16 +0000"  >&lt;p&gt;I can confirm that running the command below on the mongo router ( mongos ) does not help, a full restart of mongos is required:&lt;br/&gt;
   db.adminCommand( &lt;/p&gt;
{ flushRouterConfig : 1 }
&lt;p&gt; )&lt;/p&gt;</comment>
                            <comment id="230967" author="renctan" created="Fri, 4 Jan 2013 16:09:26 +0000"  >&lt;p&gt;Interesting, the mongos router process has been using pooled connections even before v2.0. We did a fix in &lt;a href=&quot;https://jira.mongodb.org/browse/SERVER-4706&quot; title=&quot;when a socket between mongos and mongod fails, close all connections immediately&quot; class=&quot;issue-link&quot; data-issue-key=&quot;SERVER-4706&quot;&gt;&lt;del&gt;SERVER-4706&lt;/del&gt;&lt;/a&gt; for 2.3.2 that could help mitigate the issues, but it currently only works on connections to stand alone servers (not replica sets) and new incoming client connections to mongos.&lt;/p&gt;</comment>
                            <comment id="230951" author="JIRAUSER1253508" created="Fri, 4 Jan 2013 15:47:04 +0000"  >&lt;p&gt;We are running into a similar situation in production with MongoDB V2.2.2&lt;/p&gt;

&lt;p&gt;It appears that with V2.2 the behavior in the Mongo Router process has changed from V2.0.6. In V2.0 when the Router experienced a network failure talking to a remote database server (shard master), it would return a failure message to the client and release the bad/failed connection. As of V2.2 the router process appears to be holding onto these failed connections and keeps re-using the bad connection resulting in the same error message on subsequent calls from the client to the router process.&lt;/p&gt;

&lt;p&gt;Can we change the mongo router to close/release these bad connections and not keep them around?&lt;/p&gt;

&lt;p&gt;Below is the error message we keep getting back from the mongo router process (the IP address is for the master in one of the shards) :&lt;br/&gt;
   9001: socket exception &lt;span class=&quot;error&quot;&gt;&amp;#91;SEND_ERROR&amp;#93;&lt;/span&gt; for x.x.x.x:27018&lt;/p&gt;

&lt;p&gt;Restarting the mongo router process fixes the issue. That is until a network failure occurs again and we are back to the constant send error above until we manually restart the router process.&lt;/p&gt;

&lt;p&gt;There is no information in the Mongo Router process log files about this issue. Everything is normal in the log file. The only interesting point is that the number of connections remains steady at 44, but these are probably just the client side connections, not the connections to the Mongo database servers.&lt;/p&gt;

&lt;p&gt;Any help with this issue will be greatly appreciated.&lt;/p&gt;</comment>
                            <comment id="218100" author="reynders" created="Tue, 18 Dec 2012 09:22:09 +0000"  >&lt;p&gt;Same problem observed here. Our setup is 2 shards, each shard being a replica set of 3 nodes, 1 mongos, 1 config server (which runs on the same machine as one of the shards&apos; primary) (this is dev setup btw). When restarting each node one at a time mongos starts to report this error. To be more exact the sh.status() does not report a problem but when trying to query anything we get SEND_ERROR (from the same replica set node that the config server also houses). After continuing redoing this query everything starts suddenly to work maybe 10 minutes or so after the node restart cycle.&lt;/p&gt;

&lt;p&gt;Running mongo 2.2.1&lt;/p&gt;</comment>
                            <comment id="178088" author="gabol" created="Tue, 23 Oct 2012 12:59:04 +0000"  >&lt;p&gt;Same problem. Trying to access not partitioned database through mongos&lt;/p&gt;</comment>
                            <comment id="177032" author="larry.loi" created="Fri, 19 Oct 2012 08:04:02 +0000"  >&lt;p&gt;I got the same problem, but the below 2 command unable to solve the problems.&lt;/p&gt;</comment>
                            <comment id="163780" author="matulef" created="Thu, 13 Sep 2012 04:38:53 +0000"  >&lt;p&gt;Hi Enrico,&lt;/p&gt;

&lt;p&gt;Did you get the message above from mongos, or from one of the mongod shards?  If it happens again, do you mind uploading the logs from the machine it happens on, and also the logs from mongo07.luan.local and mongo0.luan.local?  &lt;/p&gt;

&lt;p&gt;This message indicates that it&apos;s having trouble contacting one of the shards, which could happen if it has a connection that&apos;s gone stale for some reason.  If this is a mongos, you can force it to refresh its connections by typing:&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&amp;gt; db.adminCommand( { flushRouterConfig : 1 } )&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;
&lt;p&gt;or if it&apos;s a mongod you can type&lt;/p&gt;
&lt;p/&gt;
&lt;div id=&quot;syntaxplugin&quot; class=&quot;syntaxplugin&quot; style=&quot;border: 1px dashed #bbb; border-radius: 5px !important; overflow: auto; max-height: 30em;&quot;&gt;
&lt;table cellspacing=&quot;0&quot; cellpadding=&quot;0&quot; border=&quot;0&quot; width=&quot;100%&quot; style=&quot;font-size: 1em; line-height: 1.4em !important; font-weight: normal; font-style: normal; color: black;&quot;&gt;
		&lt;tbody &gt;
				&lt;tr id=&quot;syntaxplugin_code_and_gutter&quot;&gt;
						&lt;td  style=&quot; line-height: 1.4em !important; padding: 0em; vertical-align: top;&quot;&gt;
					&lt;pre style=&quot;font-size: 1em; margin: 0 10px;  margin-top: 10px;   margin-bottom: 10px;  width: auto; padding: 0;&quot;&gt;&lt;span style=&quot;color: black; font-family: &apos;Consolas&apos;, &apos;Bitstream Vera Sans Mono&apos;, &apos;Courier New&apos;, Courier, monospace !important;&quot;&gt;&amp;gt; db.adminCommand( { connPoolSync : 1 } )&lt;/span&gt;&lt;/pre&gt;
			&lt;/td&gt;
		&lt;/tr&gt;
			&lt;/tbody&gt;
&lt;/table&gt;
&lt;/div&gt;
&lt;p/&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10011">
                    <name>Depends</name>
                                            <outwardlinks description="depends on">
                                        <issuelink>
            <issuekey id="69178">SERVER-9041</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is depended on by">
                                                        </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10012">
                    <name>Related</name>
                                            <outwardlinks description="related to">
                                        <issuelink>
            <issuekey id="127616">SERVER-13352</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="68991">SERVER-9022</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is related to">
                                                        </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                <customfield id="customfield_10050" key="com.atlassian.jira.toolkit:comments">
                        <customfieldname># Replies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>50.0</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10055" key="com.atlassian.jira.ext.charting:firstresponsedate">
                        <customfieldname>Date of 1st Reply</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>Thu, 13 Sep 2012 04:38:53 +0000</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10052" key="com.atlassian.jira.toolkit:dayslastcommented">
                        <customfieldname>Days since reply</customfieldname>
                        <customfieldvalues>
                                        9 years, 29 weeks, 2 days ago
    
                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_18254" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Dependencies</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[<s><a href='https://jira.mongodb.org/browse/SERVER-9041'>SERVER-9041</a></s>]]></customfieldvalue>


                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_15850" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_10057" key="com.atlassian.jira.toolkit:lastusercommented">
                        <customfieldname>Last comment by Customer</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>true</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10056" key="com.atlassian.jira.toolkit:lastupdaterorcommenter">
                        <customfieldname>Last commenter</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>luke.bonanomi@mongodb.com</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_11151" key="com.atlassian.jira.toolkit:LastCommentDate">
                        <customfieldname>Last public comment date</customfieldname>
                        <customfieldvalues>
                            9 years, 29 weeks, 2 days ago
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10000" key="com.atlassian.jira.plugin.system.customfieldtypes:radiobuttons">
                        <customfieldname>Old_Backport</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10000"><![CDATA[No]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10032" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Operating System</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10026"><![CDATA[ALL]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_10051" key="com.atlassian.jira.toolkit:participants">
                        <customfieldname>Participants</customfieldname>
                        <customfieldvalues>
                                        <customfieldvalue>apiggott@ikanow.com</customfieldvalue>
            <customfieldvalue>alexandrov</customfieldvalue>
            <customfieldvalue>airesp</customfieldvalue>
            <customfieldvalue>gabol</customfieldvalue>
            <customfieldvalue>dan@mongodb.com</customfieldvalue>
            <customfieldvalue>disappointed</customfieldvalue>
            <customfieldvalue>eliot</customfieldvalue>
            <customfieldvalue>e.hofmann@activate.de</customfieldvalue>
            <customfieldvalue>eyllo</customfieldvalue>
            <customfieldvalue>geordios</customfieldvalue>
            <customfieldvalue>greg_10gen</customfieldvalue>
            <customfieldvalue>jblackburn</customfieldvalue>
            <customfieldvalue>jessecai</customfieldvalue>
            <customfieldvalue>reynders</customfieldvalue>
            <customfieldvalue>matulef</customfieldvalue>
            <customfieldvalue>larry.loi</customfieldvalue>
            <customfieldvalue>openmosix</customfieldvalue>
            <customfieldvalue>mbailey@gild.com</customfieldvalue>
            <customfieldvalue>noizwaves</customfieldvalue>
            <customfieldvalue>randolph@mongodb.com</customfieldvalue>
            <customfieldvalue>reidmo@gmail.com</customfieldvalue>
            <customfieldvalue>bgsosh@gmail.com</customfieldvalue>
            <customfieldvalue>vishal.katikineni@valuelabs.com</customfieldvalue>
    
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                        <customfield id="customfield_14254" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Product Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrnp3b:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                <customfield id="customfield_12550" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>2|hrhkpz:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10558" key="com.pyxis.greenhopper.jira:gh-global-rank">
                        <customfieldname>Rank (Obsolete)</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>16165</customfieldvalue>
                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_23361" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Requested By</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            <customfield id="customfield_10053" key="com.atlassian.jira.ext.charting:timeinstatus">
                        <customfieldname>Time In Status</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_22870" key="com.onresolve.jira.groovy.groovyrunner:scripted-field">
                        <customfieldname>Triagers</customfieldname>
                        <customfieldvalues>
                                

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                    <customfield id="customfield_14350" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>serverRank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>1|hrxq6f:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                    </customfields>
    </item>
</channel>
</rss>