-
Type:
Build Failure
-
Resolution: Unresolved
-
Priority:
Unknown
-
None
-
Affects Version/s: None
-
Component/s: greenerbuild
-
None
-
None
-
Python Drivers
-
None
-
None
-
None
-
None
-
None
-
None
Name of Failure:
test.test_csot.TestCsotRetryabilityTimeoutMS.test_timeoutMS_applies_to_whole_operation_not_individual_attempts-_distinct_on_collection
Link to task:
Context of when and why the failure occurred:
Stack trace:
[2025/04/07 08:33:35.970] FAILURE: pymongo.errors.OperationFailure: Failing command via 'failCommand' failpoint, full error: {'errorLabels': ['RetryableWriteError'], 'ok': 0.0, 'errmsg': "Failing command via 'failCommand' failpoint", 'code': 7, 'codeName': 'HostNotFound', '$clusterTime': {'clusterTime': Timestamp(1744039698, 6), 'signature': {'hash': b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'keyId': 0}}, 'operationTime': Timestamp(1744039698, 6)} () [2025/04/07 08:33:35.970] self = <test.test_csot.TestCsotRetryabilityTimeoutMS testMethod=test_timeoutMS_applies_to_whole_operation,_not_individual_attempts_-_distinct_on_collection> [2025/04/07 08:33:35.970] def test_case(self): [2025/04/07 08:33:35.970] > self.run_scenario(spec) [2025/04/07 08:33:35.970] test/unified_format.py:1464: [2025/04/07 08:33:35.970] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ [2025/04/07 08:33:35.970] test/unified_format.py:1390: in run_scenario [2025/04/07 08:33:35.970] return self._run_scenario(spec, uri) [2025/04/07 08:33:35.970] test/unified_format.py:1441: in _run_scenario [2025/04/07 08:33:35.970] self.run_operations(spec["operations"]) [2025/04/07 08:33:35.970] test/unified_format.py:1244: in run_operations [2025/04/07 08:33:35.970] self.run_entity_operation(op) [2025/04/07 08:33:35.970] test/unified_format.py:992: in run_entity_operation [2025/04/07 08:33:35.970] return self.process_error(exc, expect_error) [2025/04/07 08:33:35.970] test/unified_format.py:640: in process_error [2025/04/07 08:33:35.970] raise exception [2025/04/07 08:33:35.970] test/unified_format.py:983: in run_entity_operation [2025/04/07 08:33:35.970] result = cmd(**dict(arguments)) [2025/04/07 08:33:35.970] pymongo/synchronous/collection.py:3186: in distinct [2025/04/07 08:33:35.970] return self._retryable_non_cursor_read(_cmd, session, operation=_Op.DISTINCT) [2025/04/07 08:33:35.970] pymongo/synchronous/collection.py:2168: in _retryable_non_cursor_read [2025/04/07 08:33:35.970] return client._retryable_read(func, self._read_preference_for(s), s, operation) [2025/04/07 08:33:35.970] pymongo/synchronous/mongo_client.py:2005: in _retryable_read [2025/04/07 08:33:35.970] return self._retry_internal( [2025/04/07 08:33:35.970] pymongo/_csot.py:118: in csot_wrapper [2025/04/07 08:33:35.970] return func(self, *args, **kwargs) [2025/04/07 08:33:35.970] pymongo/synchronous/mongo_client.py:1961: in _retry_internal [2025/04/07 08:33:35.970] return _ClientConnectionRetryable( [2025/04/07 08:33:35.970] pymongo/synchronous/mongo_client.py:2707: in run [2025/04/07 08:33:35.970] self._check_last_error(check_csot=True) [2025/04/07 08:33:35.970] pymongo/synchronous/mongo_client.py:2795: in _check_last_error [2025/04/07 08:33:35.970] raise self._last_error [2025/04/07 08:33:35.970] pymongo/synchronous/mongo_client.py:2709: in run [2025/04/07 08:33:35.970] return self._read() if self._is_read else self._write() [2025/04/07 08:33:35.970] pymongo/synchronous/mongo_client.py:2870: in _read [2025/04/07 08:33:35.970] return self._func(self._session, self._server, conn, read_pref) # type: ignore [2025/04/07 08:33:35.970] pymongo/synchronous/collection.py:3175: in _cmd [2025/04/07 08:33:35.970] self._command( [2025/04/07 08:33:35.970] pymongo/synchronous/collection.py:621: in _command [2025/04/07 08:33:35.970] return conn.command( [2025/04/07 08:33:35.970] pymongo/synchronous/helpers.py:47: in inner [2025/04/07 08:33:35.970] return func(*args, **kwargs) [2025/04/07 08:33:35.970] pymongo/synchronous/pool.py:410: in command [2025/04/07 08:33:35.970] return command( [2025/04/07 08:33:35.970] pymongo/synchronous/network.py:212: in command [2025/04/07 08:33:35.970] helpers_shared._check_command_response( [2025/04/07 08:33:35.970] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ [2025/04/07 08:33:35.970] response = {'$clusterTime': {'clusterTime': Timestamp(1744039698, 6), 'signature': {'hash': b'\x00\x00\x00\x00\x00\x00\x00\x00\x0...00', 'keyId': 0}}, 'code': 7, 'codeName': 'HostNotFound', 'errmsg': "Failing command via 'failCommand' failpoint", ...} [2025/04/07 08:33:35.970] max_wire_version = 27, allowable_errors = None, parse_write_concern_error = True [2025/04/07 08:33:35.970] def _check_command_response( [2025/04/07 08:33:35.970] response: _DocumentOut, [2025/04/07 08:33:35.970] max_wire_version: Optional[int], [2025/04/07 08:33:35.970] allowable_errors: Optional[Container[Union[int, str]]] = None, [2025/04/07 08:33:35.970] parse_write_concern_error: bool = False, [2025/04/07 08:33:35.970] ) -> None: [2025/04/07 08:33:35.970] """Check the response to a command for errors.""" [2025/04/07 08:33:35.970] if "ok" not in response: [2025/04/07 08:33:35.970] # Server didn't recognize our message as a command. [2025/04/07 08:33:35.970] raise OperationFailure( [2025/04/07 08:33:35.970] response.get("$err"), # type: ignore[arg-type] [2025/04/07 08:33:35.970] response.get("code"), [2025/04/07 08:33:35.970] response, [2025/04/07 08:33:35.970] max_wire_version, [2025/04/07 08:33:35.970] ) [2025/04/07 08:33:35.970] [2025/04/07 08:33:35.970] if parse_write_concern_error and "writeConcernError" in response: [2025/04/07 08:33:35.970] _error = response["writeConcernError"] [2025/04/07 08:33:35.970] _labels = response.get("errorLabels") [2025/04/07 08:33:35.970] if _labels: [2025/04/07 08:33:35.970] _error.update({"errorLabels": _labels}) [2025/04/07 08:33:35.970] _raise_write_concern_error(_error) [2025/04/07 08:33:35.970] [2025/04/07 08:33:35.970] if response["ok"]: [2025/04/07 08:33:35.970] return [2025/04/07 08:33:35.970] [2025/04/07 08:33:35.970] details = response [2025/04/07 08:33:35.970] # Mongos returns the error details in a 'raw' object [2025/04/07 08:33:35.970] # for some errors. [2025/04/07 08:33:35.970] if "raw" in response: [2025/04/07 08:33:35.970] for shard in response["raw"].values(): [2025/04/07 08:33:35.970] # Grab the first non-empty raw error from a shard. [2025/04/07 08:33:35.970] if shard.get("errmsg") and not shard.get("ok"): [2025/04/07 08:33:35.970] details = shard [2025/04/07 08:33:35.970] break [2025/04/07 08:33:35.970] [2025/04/07 08:33:35.970] errmsg = details["errmsg"] [2025/04/07 08:33:35.970] code = details.get("code") [2025/04/07 08:33:35.970] [2025/04/07 08:33:35.970] # For allowable errors, only check for error messages when the code is not [2025/04/07 08:33:35.970] # included. [2025/04/07 08:33:35.970] if allowable_errors: [2025/04/07 08:33:35.970] if code is not None: [2025/04/07 08:33:35.970] if code in allowable_errors: [2025/04/07 08:33:35.970] return [2025/04/07 08:33:35.970] elif errmsg in allowable_errors: [2025/04/07 08:33:35.970] return [2025/04/07 08:33:35.970] [2025/04/07 08:33:35.970] # Server is "not primary" or "recovering" [2025/04/07 08:33:35.970] if code is not None: [2025/04/07 08:33:35.970] if code in _NOT_PRIMARY_CODES: [2025/04/07 08:33:35.970] raise NotPrimaryError(errmsg, response) [2025/04/07 08:33:35.970] elif HelloCompat.LEGACY_ERROR in errmsg or "node is recovering" in errmsg: [2025/04/07 08:33:35.970] raise NotPrimaryError(errmsg, response) [2025/04/07 08:33:35.970] [2025/04/07 08:33:35.970] # Other errors [2025/04/07 08:33:35.970] # findAndModify with upsert can raise duplicate key error [2025/04/07 08:33:35.970] if code in (11000, 11001, 12582): [2025/04/07 08:33:35.970] raise DuplicateKeyError(errmsg, code, response, max_wire_version) [2025/04/07 08:33:35.970] elif code == 50: [2025/04/07 08:33:35.970] raise ExecutionTimeout(errmsg, code, response, max_wire_version) [2025/04/07 08:33:35.970] elif code == 43: [2025/04/07 08:33:35.970] raise CursorNotFound(errmsg, code, response, max_wire_version) [2025/04/07 08:33:35.970] [2025/04/07 08:33:35.970] > raise OperationFailure(errmsg, code, response, max_wire_version) [2025/04/07 08:33:35.970] E pymongo.errors.OperationFailure: Failing command via 'failCommand' failpoint, full error: {'errorLabels': ['RetryableWriteError'], 'ok': 0.0, 'errmsg': "Failing command via 'failCommand' failpoint", 'code': 7, 'codeName': 'HostNotFound', '$clusterTime': {'clusterTime': Timestamp(1744039698, 6), 'signature': {'hash': b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 'keyId': 0}}, 'operationTime': Timestamp(1744039698, 6)} [2025/04/07 08:33:35.970] pymongo/helpers_shared.py:250: OperationFailure