web3 6.20.2__py3-none-any.whl → 7.0.0__py3-none-any.whl

web3/_utils/module_testing/module_testing_utils.py

@@ -1,17 +1,22 @@
+import asyncio
+import functools
 import pytest
-import time
 from typing import (
     TYPE_CHECKING,
     Any,
+    Callable,
     Collection,
     Dict,
     Generator,
-    Optional,
+    Literal,
     Sequence,
+    Tuple,
+    Type,
     Union,
 )
 
 from aiohttp import (
+    ClientSession,
     ClientTimeout,
 )
 from eth_typing import (
@@ -27,14 +32,10 @@ from flaky import (
 from hexbytes import (
     HexBytes,
 )
+import requests
 
-from web3._utils.compat import (
-    Literal,
-)
-from web3._utils.request import (
-    async_cache_and_return_session,
-    asyncio,
-    cache_and_return_session,
+from web3._utils.http import (
+    DEFAULT_HTTP_TIMEOUT,
 )
 from web3.types import (
     BlockData,
@@ -43,7 +44,9 @@ from web3.types import (
 
 if TYPE_CHECKING:
     from _pytest.monkeypatch import MonkeyPatch  # noqa: F401
-    from aiohttp import ClientResponse  # noqa: F401
+    from aiohttp import (  # noqa: F401
+        ClientResponse,
+    )
     from requests import Response  # noqa: F401
 
     from web3 import Web3  # noqa: F401
@@ -57,19 +60,44 @@ flaky_geth_dev_mining decorator for tests requiring a pending block
 for the duration of the test. This behavior can be flaky
 due to timing of the test running as a block is mined.
 """
-flaky_geth_dev_mining = flaky(max_runs=3)
-
-
-def mine_pending_block(w3: "Web3") -> None:
-    with pytest.warns(DeprecationWarning):
-        timeout = 10
-
-        w3.geth.miner.start()  # type: ignore
-        start = time.time()
-        while time.time() < start + timeout:
-            if len(w3.eth.get_block("pending")["transactions"]) == 0:
-                break
-        w3.geth.miner.stop()  # type: ignore
+flaky_geth_dev_mining = flaky(max_runs=3, min_passes=1)
+
+
+def flaky_with_xfail_on_exception(
+    reason: str,
+    exception: Union[Type[Exception], Tuple[Type[Exception], ...]],
+    max_runs: int = 3,
+    min_passes: int = 1,
+) -> Callable[[Any], Any]:
+    """
+    Some tests inconsistently fail hard with a particular exception and retrying
+    these tests often times does not get them "unstuck". If we've exhausted all flaky
+    retries and this expected exception is raised, `xfail` the test with the given
+    reason.
+    """
+    runs = max_runs
+
+    def decorator(func: Any) -> Any:
+        @flaky(max_runs=max_runs, min_passes=min_passes)
+        @functools.wraps(func)
+        async def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
+            nonlocal runs
+            try:
+                return await func(self, *args, **kwargs)
+            except exception:
+                # xfail the test only if the exception is raised and we have exhausted
+                # all flaky retries
+                if runs == 1:
+                    pytest.xfail(reason)
+                runs -= 1
+                pytest.fail(f"xfailed but {runs} run(s) remaining with flaky...")
+            except Exception as e:
+                # let flaky handle it
+                raise e
+
+        return wrapper
+
+    return decorator
 
 
 def assert_contains_log(
@@ -117,13 +145,13 @@ def mock_offchain_lookup_request_response(
 
         # mock response only to specified url while validating appropriate fields
         if url_from_args == mocked_request_url:
-            assert kwargs["timeout"] == 10
+            assert kwargs["timeout"] == DEFAULT_HTTP_TIMEOUT
             if http_method.upper() == "POST":
                 assert kwargs["data"] == {"data": calldata, "sender": sender}
             return MockedResponse()
 
         # else, make a normal request (no mocking)
-        session = cache_and_return_session(url_from_args)
+        session = requests.Session()
         return session.request(method=http_method.upper(), url=url_from_args, **kwargs)
 
     monkeypatch.setattr(
@@ -167,29 +195,29 @@ def async_mock_offchain_lookup_request_response(
 
         # mock response only to specified url while validating appropriate fields
         if url_from_args == mocked_request_url:
-            assert kwargs["timeout"] == ClientTimeout(10)
+            assert kwargs["timeout"] == ClientTimeout(DEFAULT_HTTP_TIMEOUT)
             if http_method.upper() == "post":
                 assert kwargs["data"] == {"data": calldata, "sender": sender}
             return AsyncMockedResponse()
 
         # else, make a normal request (no mocking)
-        session = await async_cache_and_return_session(url_from_args)
-        return await session.request(
+        session = ClientSession()
+        response = await session.request(
             method=http_method.upper(), url=url_from_args, **kwargs
         )
+        await session.close()
+        return response
 
     monkeypatch.setattr(
         f"aiohttp.ClientSession.{http_method.lower()}", _mock_specific_request
     )
 
 
-class WebsocketMessageStreamMock:
+class WebSocketMessageStreamMock:
     closed: bool = False
 
     def __init__(
-        self,
-        messages: Optional[Collection[bytes]] = None,
-        raise_exception: Optional[Exception] = None,
+        self, messages: Collection[bytes] = None, raise_exception: Exception = None
     ) -> None:
         self.queue = asyncio.Queue()  # type: ignore  # py38 issue
         for msg in messages or []:
@@ -206,12 +234,11 @@ class WebsocketMessageStreamMock:
         return self
 
     async def __anext__(self) -> bytes:
-        return await self.recv()
+        return await self.queue.get()
 
     async def recv(self) -> bytes:
         if self.raise_exception:
             raise self.raise_exception
-
         return await self.queue.get()
 
     @staticmethod

web3/_utils/module_testing/persistent_connection_provider.py

@@ -19,18 +19,35 @@ from web3.datastructures import (
     AttributeDict,
 )
 from web3.middleware import (
-    async_geth_poa_middleware,
+    ExtraDataToPOAMiddleware,
 )
 from web3.types import (
     FormattedEthSubscriptionResponse,
+    RPCEndpoint,
 )
 
 if TYPE_CHECKING:
     from web3.main import (
-        _PersistentConnectionWeb3,
+        AsyncWeb3,
     )
 
 
+SOME_BLOCK_KEYS = [
+    "number",
+    "hash",
+    "parentHash",
+    "transactionsRoot",
+    "stateRoot",
+    "receiptsRoot",
+    "size",
+    "gasLimit",
+    "gasUsed",
+    "timestamp",
+    "transactions",
+    "baseFeePerGas",
+]
+
+
 class PersistentConnectionProviderTest:
     @pytest.mark.asyncio
     @pytest.mark.parametrize(
@@ -278,7 +295,7 @@ class PersistentConnectionProviderTest:
     )
     async def test_async_eth_subscribe_mocked(
         self,
-        async_w3: "_PersistentConnectionWeb3",
+        async_w3: "AsyncWeb3",
         subscription_params: Tuple[Any, ...],
         ws_subscription_response: Dict[str, Any],
         expected_formatted_result: Any,
@@ -295,21 +312,22 @@ class PersistentConnectionProviderTest:
             ws_subscription_response, subscription=True
         )
 
-        async for msg in async_w3.ws.process_subscriptions():
+        async for msg in async_w3.socket.process_subscriptions():
             response = cast(FormattedEthSubscriptionResponse, msg)
             assert response["subscription"] == sub_id
             assert response["result"] == expected_formatted_result
 
             # only testing one message, so break here
+            await async_w3.eth.unsubscribe(sub_id)
             break
 
     @pytest.mark.asyncio
-    async def test_async_geth_poa_middleware_on_eth_subscription(
+    async def test_async_extradata_poa_middleware_on_eth_subscription(
         self,
-        async_w3: "_PersistentConnectionWeb3",
+        async_w3: "AsyncWeb3",
     ) -> None:
         async_w3.middleware_onion.inject(
-            async_geth_poa_middleware, "poa_middleware", layer=0
+            ExtraDataToPOAMiddleware, "poa_middleware", layer=0
         )
 
         sub_id = await async_w3.eth.subscribe("newHeads")
@@ -331,7 +349,7 @@ class PersistentConnectionProviderTest:
             subscription=True,
         )
 
-        async for msg in async_w3.ws.process_subscriptions():
+        async for msg in async_w3.socket.process_subscriptions():
             response = cast(FormattedEthSubscriptionResponse, msg)
             assert response.keys() == {"subscription", "result"}
             assert response["subscription"] == sub_id
@@ -348,7 +366,7 @@ class PersistentConnectionProviderTest:
     @pytest.mark.asyncio
     async def test_asyncio_gather_for_multiple_requests_matches_the_responses(
         self,
-        async_w3: "_PersistentConnectionWeb3",
+        async_w3: "AsyncWeb3",
     ) -> None:
         (
             latest,
@@ -371,22 +389,10 @@ class PersistentConnectionProviderTest:
         assert isinstance(pending, AttributeDict)
 
         # assert block values
-        some_block_keys = [
-            "number",
-            "hash",
-            "parentHash",
-            "transactionsRoot",
-            "stateRoot",
-            "receiptsRoot",
-            "size",
-            "gasLimit",
-            "gasUsed",
-            "timestamp",
-            "transactions",
-            "baseFeePerGas",
-        ]
-        assert all(k in latest.keys() for k in some_block_keys)
-        assert all(k in pending.keys() for k in some_block_keys)
+        assert latest is not None
+        assert all(k in latest.keys() for k in SOME_BLOCK_KEYS)
+        assert pending is not None
+        assert all(k in pending.keys() for k in SOME_BLOCK_KEYS)
 
         assert isinstance(block_num, int)
         assert latest["number"] == block_num
@@ -394,3 +400,28 @@ class PersistentConnectionProviderTest:
         assert isinstance(chain_id, int)
         assert isinstance(chain_id2, int)
         assert isinstance(chain_id3, int)
+
+    @pytest.mark.asyncio
+    async def test_public_socket_api(self, async_w3: "AsyncWeb3") -> None:
+        # send a request over the socket
+        await async_w3.socket.send(
+            RPCEndpoint("eth_getBlockByNumber"), ["latest", True]
+        )
+
+        # recv and validate the unprocessed response
+        response = await async_w3.socket.recv()
+        assert "id" in response, "Expected 'id' key in response."
+        assert "jsonrpc" in response, "Expected 'jsonrpc' key in response."
+        assert "result" in response, "Expected 'result' key in response."
+        assert all(k in response["result"].keys() for k in SOME_BLOCK_KEYS)
+        assert not isinstance(response["result"]["number"], int)  # assert not processed
+
+        # make a request over the socket
+        response = await async_w3.socket.make_request(
+            RPCEndpoint("eth_getBlockByNumber"), ["latest", True]
+        )
+        assert "id" in response, "Expected 'id' key in response."
+        assert "jsonrpc" in response, "Expected 'jsonrpc' key in response."
+        assert "result" in response, "Expected 'result' key in response."
+        assert all(k in response["result"].keys() for k in SOME_BLOCK_KEYS)
+        assert not isinstance(response["result"]["number"], int)  # assert not processed

web3/_utils/module_testing/utils.py

@@ -0,0 +1,258 @@
+from asyncio import (
+    iscoroutinefunction,
+)
+import copy
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Union,
+    cast,
+)
+
+from toolz import (
+    merge,
+)
+
+if TYPE_CHECKING:
+    from web3 import (  # noqa: F401
+        AsyncWeb3,
+        Web3,
+    )
+    from web3._utils.compat import (  # noqa: F401
+        Self,
+    )
+    from web3.types import (  # noqa: F401
+        AsyncMakeRequestFn,
+        MakeRequestFn,
+        RPCEndpoint,
+        RPCResponse,
+    )
+
+
+class RequestMocker:
+    """
+    Context manager to mock requests made by a web3 instance. This is meant to be used
+    via a ``request_mocker`` fixture defined within the appropriate context.
+
+    Example:
+    -------
+
+        def test_my_w3(w3, request_mocker):
+            assert w3.eth.block_number == 0
+
+            with request_mocker(w3, mock_results={"eth_blockNumber": "0x1"}):
+                assert w3.eth.block_number == 1
+
+            assert w3.eth.block_number == 0
+
+    Example with async and a mocked response object:
+    -----------------------------------------------
+
+        async def test_my_w3(async_w3, request_mocker):
+            def _iter_responses():
+                    while True:
+                        yield {"error": {"message": "transaction indexing in progress"}}
+                        yield {"error": {"message": "transaction indexing in progress"}}
+                        yield {"result": {"status": "0x1"}}
+
+            iter_responses = _iter_responses()
+
+            async with request_mocker(
+                async_w3,
+                mock_responses={
+                    "eth_getTransactionReceipt": lambda *_: next(iter_responses)
+                },
+            ):
+                # assert that the first two error responses are handled and the result
+                # is eventually returned when present
+                assert await w3.eth.get_transaction_receipt("0x1") == "0x1"
+
+
+    - ``mock_results`` is a dict mapping method names to the desired "result" object of
+        the RPC response.
+    - ``mock_errors`` is a dict mapping method names to the desired
+        "error" object of the RPC response.
+    -``mock_responses`` is a dict mapping method names to the entire RPC response
+        object. This can be useful if you wish to return an iterator which returns
+        different responses on each call to the method.
+
+    If a method name is not present in any of the dicts above, the request is made as
+    usual.
+
+    """
+
+    def __init__(
+        self,
+        w3: Union["AsyncWeb3", "Web3"],
+        mock_results: Dict[Union["RPCEndpoint", str], Any] = None,
+        mock_errors: Dict[Union["RPCEndpoint", str], Any] = None,
+        mock_responses: Dict[Union["RPCEndpoint", str], Any] = None,
+    ):
+        self.w3 = w3
+        self.mock_results = mock_results or {}
+        self.mock_errors = mock_errors or {}
+        self.mock_responses = mock_responses or {}
+        self._make_request: Union[
+            "AsyncMakeRequestFn", "MakeRequestFn"
+        ] = w3.provider.make_request
+
+    def __enter__(self) -> "Self":
+        # mypy error: Cannot assign to a method
+        self.w3.provider.make_request = self._mock_request_handler  # type: ignore[method-assign]  # noqa: E501
+        # reset request func cache to re-build request_func with mocked make_request
+        self.w3.provider._request_func_cache = (None, None)
+
+        return self
+
+    # define __exit__ with typing information
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        # mypy error: Cannot assign to a method
+        self.w3.provider.make_request = self._make_request  # type: ignore[method-assign]  # noqa: E501
+        # reset request func cache to re-build request_func with original make_request
+        self.w3.provider._request_func_cache = (None, None)
+
+    def _mock_request_handler(
+        self, method: "RPCEndpoint", params: Any
+    ) -> "RPCResponse":
+        self.w3 = cast("Web3", self.w3)
+        self._make_request = cast("MakeRequestFn", self._make_request)
+
+        if all(
+            method not in mock_dict
+            for mock_dict in (self.mock_errors, self.mock_results, self.mock_responses)
+        ):
+            return self._make_request(method, params)
+
+        request_id = (
+            next(copy.deepcopy(self.w3.provider.request_counter))
+            if hasattr(self.w3.provider, "request_counter")
+            else 1
+        )
+        response_dict = {"jsonrpc": "2.0", "id": request_id}
+
+        if method in self.mock_responses:
+            mock_return = self.mock_responses[method]
+            if callable(mock_return):
+                mock_return = mock_return(method, params)
+
+            if "result" in mock_return:
+                mock_return = {"result": mock_return["result"]}
+            elif "error" in mock_return:
+                mock_return = self._create_error_object(mock_return["error"])
+
+            mocked_response = merge(response_dict, mock_return)
+        elif method in self.mock_results:
+            mock_return = self.mock_results[method]
+            if callable(mock_return):
+                mock_return = mock_return(method, params)
+            mocked_response = merge(response_dict, {"result": mock_return})
+        elif method in self.mock_errors:
+            error = self.mock_errors[method]
+            if callable(error):
+                error = error(method, params)
+            mocked_response = merge(response_dict, self._create_error_object(error))
+        else:
+            raise Exception("Invariant: unreachable code path")
+
+        decorator = getattr(self._make_request, "_decorator", None)
+        if decorator is not None:
+            # If the original make_request was decorated, we need to re-apply
+            # the decorator to the mocked make_request. This is necessary for
+            # the request caching decorator to work properly.
+            return decorator(lambda *_: mocked_response)(
+                self.w3.provider, method, params
+            )
+        else:
+            return mocked_response
+
+    # -- async -- #
+    async def __aenter__(self) -> "Self":
+        # mypy error: Cannot assign to a method
+        self.w3.provider.make_request = self._async_mock_request_handler  # type: ignore[method-assign]  # noqa: E501
+        # reset request func cache to re-build request_func with mocked make_request
+        self.w3.provider._request_func_cache = (None, None)
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        # mypy error: Cannot assign to a method
+        self.w3.provider.make_request = self._make_request  # type: ignore[method-assign]  # noqa: E501
+        # reset request func cache to re-build request_func with original make_request
+        self.w3.provider._request_func_cache = (None, None)
+
+    async def _async_mock_request_handler(
+        self, method: "RPCEndpoint", params: Any
+    ) -> "RPCResponse":
+        self.w3 = cast("AsyncWeb3", self.w3)
+        self._make_request = cast("AsyncMakeRequestFn", self._make_request)
+
+        if all(
+            method not in mock_dict
+            for mock_dict in (self.mock_errors, self.mock_results, self.mock_responses)
+        ):
+            return await self._make_request(method, params)
+
+        request_id = (
+            next(copy.deepcopy(self.w3.provider.request_counter))
+            if hasattr(self.w3.provider, "request_counter")
+            else 1
+        )
+        response_dict = {"jsonrpc": "2.0", "id": request_id}
+
+        if method in self.mock_responses:
+            mock_return = self.mock_responses[method]
+
+            if callable(mock_return):
+                mock_return = mock_return(method, params)
+            elif iscoroutinefunction(mock_return):
+                # this is the "correct" way to mock the async make_request
+                mock_return = await mock_return(method, params)
+
+            if "result" in mock_return:
+                mock_return = {"result": mock_return["result"]}
+            elif "error" in mock_return:
+                mock_return = self._create_error_object(mock_return["error"])
+
+            mocked_result = merge(response_dict, mock_return)
+        elif method in self.mock_results:
+            mock_return = self.mock_results[method]
+            if callable(mock_return):
+                # handle callable to make things easier since we're mocking
+                mock_return = mock_return(method, params)
+            elif iscoroutinefunction(mock_return):
+                # this is the "correct" way to mock the async make_request
+                mock_return = await mock_return(method, params)
+
+            mocked_result = merge(response_dict, {"result": mock_return})
+
+        elif method in self.mock_errors:
+            error = self.mock_errors[method]
+            if callable(error):
+                error = error(method, params)
+            elif iscoroutinefunction(error):
+                error = await error(method, params)
+            mocked_result = merge(response_dict, self._create_error_object(error))
+
+        else:
+            raise Exception("Invariant: unreachable code path")
+
+        decorator = getattr(self._make_request, "_decorator", None)
+        if decorator is not None:
+            # If the original make_request was decorated, we need to re-apply
+            # the decorator to the mocked make_request. This is necessary for
+            # the request caching decorator to work properly.
+
+            async def _coro(
+                _provider: Any, _method: "RPCEndpoint", _params: Any
+            ) -> "RPCResponse":
+                return mocked_result
+
+            return await decorator(_coro)(self.w3.provider, method, params)
+        else:
+            return mocked_result
+
+    @staticmethod
+    def _create_error_object(error: Dict[str, Any]) -> Dict[str, Any]:
+        code = error.get("code", -32000)
+        message = error.get("message", "Mocked error")
+        return {"error": merge({"code": code, "message": message}, error)}