[2024/11/19 14:34:29.150] self = AsyncConnection(<ssl.SSLSocket [closed] fd=-1, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=6>) CLOSED at 483634328
 [2024/11/19 14:34:29.150] dbname = 'admin'
 [2024/11/19 14:34:29.150] spec = {'$clusterTime': {'clusterTime': Timestamp(1732054833, 12), 'signature': {'hash': b'\x00\x00\x00\x00\x00\x00\x00\x00\x...ureFailPoint': 'failCommand', 'lsid': {'id': Binary(b'\xca\x04\x17O\xf0\xdcH\xb7\x94\xee\xd7\xf9\xd2\xc4/\t', 4)}, ...}
 [2024/11/19 14:34:29.150] read_preference = Primary()
 [2024/11/19 14:34:29.150] codec_options = CodecOptions(document_class=dict, tz_aware=False, uuid_representation=UuidRepresentation.UNSPECIFIED, unicode_decode_e...ne, type_registry=TypeRegistry(type_codecs=[], fallback_encoder=None), datetime_conversion=DatetimeConversion.DATETIME)
 [2024/11/19 14:34:29.150] check = True, allowable_errors = None, read_concern = None, write_concern = None
 [2024/11/19 14:34:29.150] parse_write_concern_error = False, collation = None
 [2024/11/19 14:34:29.150] session = <pymongo.asynchronous.client_session.AsyncClientSession object at 0x000000001e18cd40>
 [2024/11/19 14:34:29.150] client = AsyncMongoClient(host=['localhost:27017'], document_class=dict, tz_aware=False, connect=True, tls=True, tlscertificate...en/x509gen/client.pem', tlscafile='/data/mci/7e44d2809b7b3baa74c39e7067df407b/drivers-tools/.evergreen/x509gen/ca.pem')
 [2024/11/19 14:34:29.150] retryable_write = False, publish_events = True, user_fields = None
 [2024/11/19 14:34:29.150] exhaust_allowed = False
 [2024/11/19 14:34:29.150]     @_handle_reauth
 [2024/11/19 14:34:29.150]     async def command(
 [2024/11/19 14:34:29.150]         self,
 [2024/11/19 14:34:29.150]         dbname: str,
 [2024/11/19 14:34:29.150]         spec: MutableMapping[str, Any],
 [2024/11/19 14:34:29.150]         read_preference: _ServerMode = ReadPreference.PRIMARY,
 [2024/11/19 14:34:29.150]         codec_options: CodecOptions = DEFAULT_CODEC_OPTIONS,
 [2024/11/19 14:34:29.150]         check: bool = True,
 [2024/11/19 14:34:29.150]         allowable_errors: Optional[Sequence[Union[str, int]]] = None,
 [2024/11/19 14:34:29.150]         read_concern: Optional[ReadConcern] = None,
 [2024/11/19 14:34:29.150]         write_concern: Optional[WriteConcern] = None,
 [2024/11/19 14:34:29.150]         parse_write_concern_error: bool = False,
 [2024/11/19 14:34:29.150]         collation: Optional[_CollationIn] = None,
 [2024/11/19 14:34:29.150]         session: Optional[AsyncClientSession] = None,
 [2024/11/19 14:34:29.150]         client: Optional[AsyncMongoClient] = None,
 [2024/11/19 14:34:29.150]         retryable_write: bool = False,
 [2024/11/19 14:34:29.150]         publish_events: bool = True,
 [2024/11/19 14:34:29.150]         user_fields: Optional[Mapping[str, Any]] = None,
 [2024/11/19 14:34:29.150]         exhaust_allowed: bool = False,
 [2024/11/19 14:34:29.150]     ) -> dict[str, Any]:
 [2024/11/19 14:34:29.150]         """Execute a command or raise an error.
 [2024/11/19 14:34:29.150]     
 [2024/11/19 14:34:29.150]         :param dbname: name of the database on which to run the command
 [2024/11/19 14:34:29.150]         :param spec: a command document as a dict, SON, or mapping object
 [2024/11/19 14:34:29.150]         :param read_preference: a read preference
 [2024/11/19 14:34:29.150]         :param codec_options: a CodecOptions instance
 [2024/11/19 14:34:29.150]         :param check: raise OperationFailure if there are errors
 [2024/11/19 14:34:29.150]         :param allowable_errors: errors to ignore if `check` is True
 [2024/11/19 14:34:29.150]         :param read_concern: The read concern for this command.
 [2024/11/19 14:34:29.150]         :param write_concern: The write concern for this command.
 [2024/11/19 14:34:29.150]         :param parse_write_concern_error: Whether to parse the
 [2024/11/19 14:34:29.150]             ``writeConcernError`` field in the command response.
 [2024/11/19 14:34:29.150]         :param collation: The collation for this command.
 [2024/11/19 14:34:29.150]         :param session: optional AsyncClientSession instance.
 [2024/11/19 14:34:29.150]         :param client: optional AsyncMongoClient for gossipping $clusterTime.
 [2024/11/19 14:34:29.150]         :param retryable_write: True if this command is a retryable write.
 [2024/11/19 14:34:29.150]         :param publish_events: Should we publish events for this command?
 [2024/11/19 14:34:29.150]         :param user_fields: Response fields that should be decoded
 [2024/11/19 14:34:29.150]             using the TypeDecoders from codec_options, passed to
 [2024/11/19 14:34:29.150]             bson._decode_all_selective.
 [2024/11/19 14:34:29.150]         """
 [2024/11/19 14:34:29.150]         self.validate_session(client, session)
 [2024/11/19 14:34:29.150]         session = _validate_session_write_concern(session, write_concern)
 [2024/11/19 14:34:29.150]     
 [2024/11/19 14:34:29.150]         # Ensure command name remains in first place.
 [2024/11/19 14:34:29.150]         if not isinstance(spec, ORDERED_TYPES):  # type:ignore[arg-type]
 [2024/11/19 14:34:29.150]             spec = dict(spec)
 [2024/11/19 14:34:29.150]     
 [2024/11/19 14:34:29.150]         if not (write_concern is None or write_concern.acknowledged or collation is None):
 [2024/11/19 14:34:29.150]             raise ConfigurationError("Collation is unsupported for unacknowledged writes.")
 [2024/11/19 14:34:29.150]     
 [2024/11/19 14:34:29.150]         self.add_server_api(spec)
 [2024/11/19 14:34:29.150]         if session:
 [2024/11/19 14:34:29.150]             session._apply_to(spec, retryable_write, read_preference, self)
 [2024/11/19 14:34:29.150]         self.send_cluster_time(spec, session, client)
 [2024/11/19 14:34:29.150]         listeners = self.listeners if publish_events else None
 [2024/11/19 14:34:29.150]         unacknowledged = bool(write_concern and not write_concern.acknowledged)
 [2024/11/19 14:34:29.150]         if self.op_msg_enabled:
 [2024/11/19 14:34:29.150]             self._raise_if_not_writable(unacknowledged)
 [2024/11/19 14:34:29.150]         try:
 [2024/11/19 14:34:29.150] >           return await command(
 [2024/11/19 14:34:29.150]                 self,
 [2024/11/19 14:34:29.150]                 dbname,
 [2024/11/19 14:34:29.150]                 spec,
 [2024/11/19 14:34:29.150]                 self.is_mongos,
 [2024/11/19 14:34:29.150]                 read_preference,
 [2024/11/19 14:34:29.150]                 codec_options,
 [2024/11/19 14:34:29.150]                 session,
 [2024/11/19 14:34:29.150]                 client,
 [2024/11/19 14:34:29.150]                 check,
 [2024/11/19 14:34:29.150]                 allowable_errors,
 [2024/11/19 14:34:29.150]                 self.address,
 [2024/11/19 14:34:29.150]                 listeners,
 [2024/11/19 14:34:29.150]                 self.max_bson_size,
 [2024/11/19 14:34:29.150]                 read_concern,
 [2024/11/19 14:34:29.150]                 parse_write_concern_error=parse_write_concern_error,
 [2024/11/19 14:34:29.150]                 collation=collation,
 [2024/11/19 14:34:29.150]                 compression_ctx=self.compression_context,
 [2024/11/19 14:34:29.150]                 use_op_msg=self.op_msg_enabled,
 [2024/11/19 14:34:29.150]                 unacknowledged=unacknowledged,
 [2024/11/19 14:34:29.150]                 user_fields=user_fields,
 [2024/11/19 14:34:29.150]                 exhaust_allowed=exhaust_allowed,
 [2024/11/19 14:34:29.150]                 write_concern=write_concern,
 [2024/11/19 14:34:29.150]             )
 [2024/11/19 14:34:29.150] pymongo/asynchronous/pool.py:536: 
 [2024/11/19 14:34:29.150] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
 [2024/11/19 14:34:29.150] pymongo/asynchronous/network.py:203: in command
 [2024/11/19 14:34:29.150]     reply = await receive_message(conn, request_id)
 [2024/11/19 14:34:29.150] pymongo/asynchronous/network.py:315: in receive_message
 [2024/11/19 14:34:29.150]     length, _, response_to, op_code = _UNPACK_HEADER(await async_receive_data(conn, 16, deadline))
 [2024/11/19 14:34:29.150] pymongo/network_layer.py:279: in async_receive_data
 [2024/11/19 14:34:29.150]     return read_task.result()
 [2024/11/19 14:34:29.150] /opt/python/pypy3.10/lib/pypy3.10/asyncio/futures.py:201: in result
 [2024/11/19 14:34:29.150]     raise self._exception.with_traceback(self._exception_tb)
 [2024/11/19 14:34:29.150] /opt/python/pypy3.10/lib/pypy3.10/asyncio/tasks.py:232: in __step
 [2024/11/19 14:34:29.150]     result = coro.send(None)
 [2024/11/19 14:34:29.150] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
 [2024/11/19 14:34:29.150] conn = <ssl.SSLSocket [closed] fd=-1, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=6>
 [2024/11/19 14:34:29.150] length = 16
 [2024/11/19 14:34:29.150] loop = <_UnixSelectorEventLoop running=False closed=True debug=True>
 [2024/11/19 14:34:29.150] once = False
 [2024/11/19 14:34:29.150]     async def _async_receive_ssl(
 [2024/11/19 14:34:29.150]         conn: _sslConn, length: int, loop: AbstractEventLoop, once: Optional[bool] = False
 [2024/11/19 14:34:29.150]     ) -> memoryview:
 [2024/11/19 14:34:29.150]         mv = memoryview(bytearray(length))
 [2024/11/19 14:34:29.150]         total_read = 0
 [2024/11/19 14:34:29.150]     
 [2024/11/19 14:34:29.150]         def _is_ready(fut: Future) -> None:
 [2024/11/19 14:34:29.150]             if fut.done():
 [2024/11/19 14:34:29.150]                 return
 [2024/11/19 14:34:29.150]             fut.set_result(None)
 [2024/11/19 14:34:29.150]     
 [2024/11/19 14:34:29.150]         while total_read < length:
 [2024/11/19 14:34:29.150]             try:
 [2024/11/19 14:34:29.150]                 read = conn.recv_into(mv[total_read:])
 [2024/11/19 14:34:29.150]                 if read == 0:
 [2024/11/19 14:34:29.150] >                   raise OSError("connection closed")
 [2024/11/19 14:34:29.150] E                   OSError: connection closed
 [2024/11/19 14:34:29.150] pymongo/network_layer.py:147: OSError
 [2024/11/19 14:34:29.150] The above exception was the direct cause of the following exception:
 [2024/11/19 14:34:29.150] self = <test.asynchronous.test_retryable_reads.TestRetryableReads testMethod=test_retryable_reads_in_sharded_cluster_multiple_available>
 [2024/11/19 14:34:29.150]     @async_client_context.require_multiple_mongoses
 [2024/11/19 14:34:29.150]     @async_client_context.require_failCommand_fail_point
 [2024/11/19 14:34:29.150]     async def test_retryable_reads_in_sharded_cluster_multiple_available(self):
 [2024/11/19 14:34:29.150]         fail_command = {
 [2024/11/19 14:34:29.150]             "configureFailPoint": "failCommand",
 [2024/11/19 14:34:29.150]             "mode": {"times": 1},
 [2024/11/19 14:34:29.150]             "data": {
 [2024/11/19 14:34:29.150]                 "failCommands": ["find"],
 [2024/11/19 14:34:29.150]                 "closeConnection": True,
 [2024/11/19 14:34:29.150]                 "appName": "retryableReadTest",
 [2024/11/19 14:34:29.150]             },
 [2024/11/19 14:34:29.150]         }
 [2024/11/19 14:34:29.150]     
 [2024/11/19 14:34:29.150]         mongos_clients = []
 [2024/11/19 14:34:29.150]     
 [2024/11/19 14:34:29.150]         for mongos in async_client_context.mongos_seeds().split(","):
 [2024/11/19 14:34:29.150]             client = await self.async_rs_or_single_client(mongos)
 [2024/11/19 14:34:29.150]             await async_set_fail_point(client, fail_command)
 [2024/11/19 14:34:29.150]             mongos_clients.append(client)
 [2024/11/19 14:34:29.150]     
 [2024/11/19 14:34:29.150]         listener = OvertCommandListener()
 [2024/11/19 14:34:29.150]         client = await self.async_rs_or_single_client(
 [2024/11/19 14:34:29.150]             async_client_context.mongos_seeds(),
 [2024/11/19 14:34:29.150]             appName="retryableReadTest",
 [2024/11/19 14:34:29.150]             event_listeners=[listener],
 [2024/11/19 14:34:29.150]             retryReads=True,
 [2024/11/19 14:34:29.150]         )
 [2024/11/19 14:34:29.150]     
 [2024/11/19 14:34:29.150] >       async with self.fail_point(fail_command):
 [2024/11/19 14:34:29.150] test/asynchronous/test_retryable_reads.py:177: 
 [2024/11/19 14:34:29.150] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
 [2024/11/19 14:34:29.150] /opt/python/pypy3.10/lib/pypy3.10/contextlib.py:206: in __aexit__
 [2024/11/19 14:34:29.150]     await anext(self.gen)
 [2024/11/19 14:34:29.150] test/asynchronous/__init__.py:893: in fail_point
 [2024/11/19 14:34:29.150]     await async_client_context.client.admin.command(
 [2024/11/19 14:34:29.150] pymongo/_csot.py:109: in csot_wrapper
 [2024/11/19 14:34:29.150]     return await func(self, *args, **kwargs)
 [2024/11/19 14:34:29.150] pymongo/asynchronous/database.py:932: in command
 [2024/11/19 14:34:29.150]     return await self._command(
 [2024/11/19 14:34:29.150] pymongo/asynchronous/database.py:770: in _command
 [2024/11/19 14:34:29.150]     return await conn.command(
 [2024/11/19 14:34:29.150] pymongo/asynchronous/helpers.py:45: in inner
 [2024/11/19 14:34:29.150]     return await func(*args, **kwargs)
 [2024/11/19 14:34:29.150] pymongo/asynchronous/pool.py:564: in command
 [2024/11/19 14:34:29.150]     self._raise_connection_failure(error)
 [2024/11/19 14:34:29.150] pymongo/asynchronous/pool.py:765: in _raise_connection_failure
 [2024/11/19 14:34:29.150]     _raise_connection_failure(self.address, error, timeout_details=details)
 [2024/11/19 14:34:29.150] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
 [2024/11/19 14:34:29.150] address = ('localhost', 27017), error = OSError('connection closed')
 [2024/11/19 14:34:29.150] msg_prefix = None, timeout_details = {'connectTimeoutMS': 20000.0}
 [2024/11/19 14:34:29.150]     def _raise_connection_failure(
 [2024/11/19 14:34:29.150]         address: Any,
 [2024/11/19 14:34:29.150]         error: Exception,
 [2024/11/19 14:34:29.150]         msg_prefix: Optional[str] = None,
 [2024/11/19 14:34:29.150]         timeout_details: Optional[dict[str, float]] = None,
 [2024/11/19 14:34:29.150]     ) -> NoReturn:
 [2024/11/19 14:34:29.150]         """Convert a socket.error to ConnectionFailure and raise it."""
 [2024/11/19 14:34:29.150]         host, port = address
 [2024/11/19 14:34:29.150]         # If connecting to a Unix socket, port will be None.
 [2024/11/19 14:34:29.150]         if port is not None:
 [2024/11/19 14:34:29.150]             msg = "%s:%d: %s" % (host, port, error)
 [2024/11/19 14:34:29.150]         else:
 [2024/11/19 14:34:29.150]             msg = f"{host}: {error}"
 [2024/11/19 14:34:29.150]         if msg_prefix:
 [2024/11/19 14:34:29.150]             msg = msg_prefix + msg
 [2024/11/19 14:34:29.150]         if "configured timeouts" not in msg:
 [2024/11/19 14:34:29.150]             msg += format_timeout_details(timeout_details)
 [2024/11/19 14:34:29.150]         if isinstance(error, socket.timeout):
 [2024/11/19 14:34:29.150]             raise NetworkTimeout(msg) from error
 [2024/11/19 14:34:29.150]         elif isinstance(error, SSLError) and "timed out" in str(error):
 [2024/11/19 14:34:29.150]             # Eventlet does not distinguish TLS network timeouts from other
 [2024/11/19 14:34:29.150]             # SSLErrors (https://github.com/eventlet/eventlet/issues/692).
 [2024/11/19 14:34:29.150]             # Luckily, we can work around this limitation because the phrase
 [2024/11/19 14:34:29.150]             # 'timed out' appears in all the timeout related SSLErrors raised.
 [2024/11/19 14:34:29.150]             raise NetworkTimeout(msg) from error
 [2024/11/19 14:34:29.150]         else:
 [2024/11/19 14:34:29.150] >           raise AutoReconnect(msg) from error
 [2024/11/19 14:34:29.150] E           pymongo.errors.AutoReconnect: localhost:27017: connection closed (configured timeouts: connectTimeoutMS: 20000.0ms)
 [2024/11/19 14:34:29.150] pymongo/asynchronous/pool.py:211: AutoReconnect