web3 6.20.3__py3-none-any.whl → 7.0.0__py3-none-any.whl

web3/providers/{websocket → persistent}/request_processor.py

@@ -15,12 +15,20 @@ from typing import (
     Union,
 )
 
+from eth_utils.toolz import (
+    compose,
+)
+
+from web3._utils.batching import (
+    BATCH_REQUEST_ID,
+)
 from web3._utils.caching import (
     RequestInformation,
     generate_cache_key,
 )
 from web3.exceptions import (
     TaskNotRunning,
+    Web3ValueError,
 )
 from web3.types import (
     RPCEndpoint,
@@ -71,17 +79,14 @@ class RequestProcessor:
         self,
         provider: "PersistentConnectionProvider",
         subscription_response_queue_size: int = 500,
-        request_information_cache_size: int = 500,
     ) -> None:
         self._provider = provider
 
+        self._request_information_cache: SimpleCache = SimpleCache(500)
         self._request_response_cache: SimpleCache = SimpleCache(500)
         self._subscription_response_queue: TaskReliantQueue[
             Union[RPCResponse, TaskNotRunning]
         ] = TaskReliantQueue(maxsize=subscription_response_queue_size)
-        self._request_information_cache: SimpleCache = SimpleCache(
-            request_information_cache_size
-        )
 
     @property
     def active_subscriptions(self) -> Dict[str, Any]:
@@ -97,14 +102,35 @@ class RequestProcessor:
         self,
         method: RPCEndpoint,
         params: Any,
-        response_formatters: Tuple[Callable[..., Any], ...],
-    ) -> str:
-        # copy the request counter and find the next request id without incrementing
-        # since this is done when / if the request is successfully sent
-        request_id = next(copy(self._provider.request_counter))
-        cache_key = generate_cache_key(request_id)
+        response_formatters: Tuple[
+            Union[Dict[str, Callable[..., Any]], Callable[..., Any]],
+            Callable[..., Any],
+            Callable[..., Any],
+        ],
+    ) -> Optional[str]:
+        cached_requests_key = generate_cache_key((method, params))
+        if cached_requests_key in self._provider._request_cache._data:
+            cached_response = self._provider._request_cache._data[cached_requests_key]
+            cached_response_id = cached_response.get("id")
+            cache_key = generate_cache_key(cached_response_id)
+            if cache_key in self._request_information_cache:
+                self._provider.logger.debug(
+                    "This is a cached request, not caching request info because it is "
+                    f"not unique:\n    method={method},\n    params={params}"
+                )
+                return None
+
+        if self._provider._is_batching:
+            # the _batch_request_counter is set when entering the context manager
+            current_request_id = self._provider._batch_request_counter
+            self._provider._batch_request_counter += 1
+        else:
+            # copy the request counter and find the next request id without incrementing
+            # since this is done when / if the request is successfully sent
+            current_request_id = next(copy(self._provider.request_counter))
+        cache_key = generate_cache_key(current_request_id)
 
-        self._bump_cache_if_key_present(cache_key, request_id)
+        self._bump_cache_if_key_present(cache_key, current_request_id)
 
         request_info = RequestInformation(
             method,
@@ -112,19 +138,13 @@ class RequestProcessor:
             response_formatters,
         )
         self._provider.logger.debug(
-            f"Caching request info:\n    request_id={request_id},\n"
+            f"Caching request info:\n    request_id={current_request_id},\n"
             f"    cache_key={cache_key},\n    request_info={request_info.__dict__}"
         )
         self._request_information_cache.cache(
             cache_key,
             request_info,
         )
-        if self._request_information_cache.is_full():
-            self._provider.logger.warning(
-                "Request information cache is full. This may result in unexpected "
-                "behavior. Consider increasing the ``request_information_cache_size`` "
-                "on the provider."
-            )
         return cache_key
 
     def _bump_cache_if_key_present(self, cache_key: str, request_id: int) -> None:
@@ -168,9 +188,9 @@ class RequestProcessor:
     ) -> RequestInformation:
         if "method" in response and response["method"] == "eth_subscription":
             if "params" not in response:
-                raise ValueError("Subscription response must have params field")
+                raise Web3ValueError("Subscription response must have params field")
             if "subscription" not in response["params"]:
-                raise ValueError(
+                raise Web3ValueError(
                     "Subscription response params must have subscription field"
                 )
 
@@ -182,15 +202,23 @@ class RequestProcessor:
                 # i.e. subscription request information remains in the cache
                 self._request_information_cache.get_cache_entry(cache_key)
             )
-
         else:
-            # retrieve the request info from the cache using the request id
+            # retrieve the request info from the cache using the response id
             cache_key = generate_cache_key(response["id"])
-            request_info = (
-                # pop the request info from the cache since we don't need to keep it
-                # this keeps the cache size bounded
-                self.pop_cached_request_information(cache_key)
-            )
+            if response in self._provider._request_cache._data.values():
+                request_info = (
+                    # don't pop the request info from the cache, since we need to keep
+                    # it to process future responses
+                    # i.e. request information remains in the cache
+                    self._request_information_cache.get_cache_entry(cache_key)
+                )
+            else:
+                request_info = (
+                    # pop the request info from the cache since we don't need to keep it
+                    # this keeps the cache size bounded
+                    self.pop_cached_request_information(cache_key)
+                )
+
             if (
                 request_info is not None
                 and request_info.method == "eth_unsubscribe"
@@ -204,6 +232,33 @@ class RequestProcessor:
 
         return request_info
 
+    def append_result_formatter_for_request(
+        self, request_id: int, result_formatter: Callable[..., Any]
+    ) -> None:
+        cache_key = generate_cache_key(request_id)
+        cached_request_info_for_id: RequestInformation = (
+            self._request_information_cache.get_cache_entry(cache_key)
+        )
+        if cached_request_info_for_id is not None:
+            (
+                current_result_formatters,
+                error_formatters,
+                null_result_formatters,
+            ) = cached_request_info_for_id.response_formatters
+            cached_request_info_for_id.response_formatters = (
+                compose(
+                    result_formatter,
+                    current_result_formatters,
+                ),
+                error_formatters,
+                null_result_formatters,
+            )
+        else:
+            self._provider.logger.debug(
+                f"No cached request info for response id `{request_id}`. Cannot "
+                f"append response formatter for response."
+            )
+
     def append_middleware_response_processor(
         self,
         response: RPCResponse,
@@ -242,12 +297,22 @@ class RequestProcessor:
                     "Subscription queue is full. Waiting for provider to consume "
                     "messages before caching."
                 )
+                self._provider._listen_event.clear()
                 await self._provider._listen_event.wait()
 
             self._provider.logger.debug(
                 f"Caching subscription response:\n    response={raw_response}"
             )
             await self._subscription_response_queue.put(raw_response)
+        elif isinstance(raw_response, list):
+            # Since only one batch should be in the cache at all times, we use a
+            # constant cache key for the batch response.
+            cache_key = generate_cache_key(BATCH_REQUEST_ID)
+            self._provider.logger.debug(
+                f"Caching batch response:\n    cache_key={cache_key},\n"
+                f"    response={raw_response}"
+            )
+            self._request_response_cache.cache(cache_key, raw_response)
         else:
             response_id = raw_response.get("id")
             cache_key = generate_cache_key(response_id)
@@ -288,7 +353,7 @@ class RequestProcessor:
             )
         else:
             if not cache_key:
-                raise ValueError(
+                raise Web3ValueError(
                     "Must provide cache key when popping a non-subscription response."
                 )
 
@@ -305,10 +370,7 @@ class RequestProcessor:
     # request processor class methods
 
     def clear_caches(self) -> None:
-        """
-        Clear the request processor caches.
-        """
-
+        """Clear the request processor caches."""
         self._request_information_cache.clear()
         self._request_response_cache.clear()
         self._subscription_response_queue = TaskReliantQueue(

web3/providers/persistent/utils.py

@@ -0,0 +1,43 @@
+import functools
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+)
+
+from web3.exceptions import (
+    Web3ValidationError,
+)
+from web3.providers import (
+    PersistentConnectionProvider,
+)
+
+if TYPE_CHECKING:
+    from web3.main import (  # noqa: F401
+        AsyncWeb3,
+    )
+
+
+def persistent_connection_provider_method(message: str = None) -> Callable[..., Any]:
+    """
+    Decorator that raises an exception if the provider is not an instance of
+    ``PersistentConnectionProvider``.
+    """
+
+    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+        @functools.wraps(func)
+        def inner(self: "AsyncWeb3", *args: Any, **kwargs: Any) -> Any:
+            nonlocal message
+            if message is None:
+                message = (
+                    f"``{func.__name__}`` can only be called on a "
+                    "``PersistentConnectionProvider`` instance."
+                )
+
+            if not isinstance(self.provider, PersistentConnectionProvider):
+                raise Web3ValidationError(message)
+            return func(self, *args, **kwargs)
+
+        return inner
+
+    return decorator

web3/providers/{websocket/websocket_v2.py → persistent/websocket.py}

@@ -15,14 +15,19 @@ from eth_typing import (
 from toolz import (
     merge,
 )
+from websockets import (
+    WebSocketClientProtocol,
+)
 from websockets.client import (
     connect,
 )
 from websockets.exceptions import (
+    ConnectionClosedOK,
     WebSocketException,
 )
 
 from web3.exceptions import (
+    PersistentConnectionClosedOK,
     ProviderConnectionError,
     Web3ValidationError,
 )
@@ -30,7 +35,6 @@ from web3.providers.persistent import (
     PersistentConnectionProvider,
 )
 from web3.types import (
-    RPCEndpoint,
     RPCResponse,
 )
 
@@ -51,15 +55,16 @@ def get_default_endpoint() -> URI:
     return URI(os.environ.get("WEB3_WS_PROVIDER_URI", "ws://127.0.0.1:8546"))
 
 
-class WebsocketProviderV2(PersistentConnectionProvider):
-    logger = logging.getLogger("web3.providers.WebsocketProviderV2")
+class WebSocketProvider(PersistentConnectionProvider):
+    logger = logging.getLogger("web3.providers.WebSocketProvider")
     is_async: bool = True
 
+    _ws: Optional[WebSocketClientProtocol] = None
+
     def __init__(
         self,
         endpoint_uri: Optional[Union[URI, str]] = None,
         websocket_kwargs: Optional[Dict[str, Any]] = None,
-        silence_listener_task_exceptions: bool = False,
         # `PersistentConnectionProvider` kwargs can be passed through
         **kwargs: Any,
     ) -> None:
@@ -88,9 +93,7 @@ class WebsocketProviderV2(PersistentConnectionProvider):
 
         self.websocket_kwargs = merge(DEFAULT_WEBSOCKET_KWARGS, websocket_kwargs or {})
 
-        super().__init__(
-            silence_listener_task_exceptions=silence_listener_task_exceptions, **kwargs
-        )
+        super().__init__(**kwargs)
 
     def __str__(self) -> str:
         return f"WebSocket connection: {self.endpoint_uri}"
@@ -110,17 +113,7 @@ class WebsocketProviderV2(PersistentConnectionProvider):
                 ) from e
             return False
 
-    async def _provider_specific_connect(self) -> None:
-        self._ws = await connect(self.endpoint_uri, **self.websocket_kwargs)
-
-    async def _provider_specific_disconnect(self) -> None:
-        if self._ws is not None and not self._ws.closed:
-            await self._ws.close()
-            self._ws = None
-
-    async def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
-        request_data = self.encode_rpc_request(method, params)
-
+    async def socket_send(self, request_data: bytes) -> None:
         if self._ws is None:
             raise ProviderConnectionError(
                 "Connection to websocket has not been initiated for the provider."
@@ -130,16 +123,24 @@ class WebsocketProviderV2(PersistentConnectionProvider):
             self._ws.send(request_data), timeout=self.request_timeout
         )
 
-        current_request_id = json.loads(request_data)["id"]
-        response = await self._get_response_for_request_id(current_request_id)
+    async def socket_recv(self) -> RPCResponse:
+        raw_response = await self._ws.recv()
+        return json.loads(raw_response)
+
+    # -- private methods -- #
+
+    async def _provider_specific_connect(self) -> None:
+        self._ws = await connect(self.endpoint_uri, **self.websocket_kwargs)
 
-        return response
+    async def _provider_specific_disconnect(self) -> None:
+        if self._ws is not None and not self._ws.closed:
+            await self._ws.close()
+            self._ws = None
 
-    async def _provider_specific_message_listener(self) -> None:
-        while True:
-            raw_message = await self._ws.recv()
-            response = json.loads(raw_message)
-            subscription = response.get("method") == "eth_subscription"
-            await self._request_processor.cache_raw_response(
-                response, subscription=subscription
+    async def _provider_specific_socket_reader(self) -> RPCResponse:
+        try:
+            return await self.socket_recv()
+        except ConnectionClosedOK:
+            raise PersistentConnectionClosedOK(
+                user_message="WebSocket connection received `ConnectionClosedOK`."
             )

web3/providers/rpc/__init__.py

@@ -0,0 +1,11 @@
+from .async_rpc import (
+    AsyncHTTPProvider,
+)
+from .rpc import (
+    HTTPProvider,
+)
+
+__all__ = [
+    "AsyncHTTPProvider",
+    "HTTPProvider",
+]

web3/providers/rpc/async_rpc.py

@@ -0,0 +1,171 @@
+import asyncio
+import logging
+from typing import (
+    Any,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+    Union,
+    cast,
+)
+
+from aiohttp import (
+    ClientError,
+    ClientSession,
+)
+from eth_typing import (
+    URI,
+)
+from eth_utils import (
+    to_dict,
+)
+
+from web3._utils.empty import (
+    Empty,
+    empty,
+)
+from web3._utils.http import (
+    construct_user_agent,
+)
+from web3.types import (
+    RPCEndpoint,
+    RPCResponse,
+)
+
+from ..._utils.batching import (
+    sort_batch_response_by_response_ids,
+)
+from ..._utils.caching import (
+    async_handle_request_caching,
+)
+from ..._utils.http_session_manager import (
+    HTTPSessionManager,
+)
+from ..async_base import (
+    AsyncJSONBaseProvider,
+)
+from .utils import (
+    ExceptionRetryConfiguration,
+    check_if_retry_on_failure,
+)
+
+
+class AsyncHTTPProvider(AsyncJSONBaseProvider):
+    logger = logging.getLogger("web3.providers.AsyncHTTPProvider")
+    endpoint_uri = None
+    _request_kwargs = None
+
+    def __init__(
+        self,
+        endpoint_uri: Optional[Union[URI, str]] = None,
+        request_kwargs: Optional[Any] = None,
+        exception_retry_configuration: Optional[
+            Union[ExceptionRetryConfiguration, Empty]
+        ] = empty,
+        **kwargs: Any,
+    ) -> None:
+        self._request_session_manager = HTTPSessionManager()
+
+        if endpoint_uri is None:
+            self.endpoint_uri = (
+                self._request_session_manager.get_default_http_endpoint()
+            )
+        else:
+            self.endpoint_uri = URI(endpoint_uri)
+
+        self._request_kwargs = request_kwargs or {}
+        self._exception_retry_configuration = exception_retry_configuration
+
+        super().__init__(**kwargs)
+
+    async def cache_async_session(self, session: ClientSession) -> ClientSession:
+        return await self._request_session_manager.async_cache_and_return_session(
+            self.endpoint_uri, session
+        )
+
+    def __str__(self) -> str:
+        return f"RPC connection {self.endpoint_uri}"
+
+    @property
+    def exception_retry_configuration(self) -> ExceptionRetryConfiguration:
+        if isinstance(self._exception_retry_configuration, Empty):
+            self._exception_retry_configuration = ExceptionRetryConfiguration(
+                errors=(ClientError, TimeoutError)
+            )
+        return self._exception_retry_configuration
+
+    @exception_retry_configuration.setter
+    def exception_retry_configuration(
+        self, value: Union[ExceptionRetryConfiguration, Empty]
+    ) -> None:
+        self._exception_retry_configuration = value
+
+    @to_dict
+    def get_request_kwargs(self) -> Iterable[Tuple[str, Any]]:
+        if "headers" not in self._request_kwargs:
+            yield "headers", self.get_request_headers()
+        yield from self._request_kwargs.items()
+
+    def get_request_headers(self) -> Dict[str, str]:
+        return {
+            "Content-Type": "application/json",
+            "User-Agent": construct_user_agent(type(self)),
+        }
+
+    async def _make_request(self, method: RPCEndpoint, request_data: bytes) -> bytes:
+        """
+        If exception_retry_configuration is set, retry on failure; otherwise, make
+        the request without retrying.
+        """
+        if (
+            self.exception_retry_configuration is not None
+            and check_if_retry_on_failure(
+                method, self.exception_retry_configuration.method_allowlist
+            )
+        ):
+            for i in range(self.exception_retry_configuration.retries):
+                try:
+                    return await self._request_session_manager.async_make_post_request(
+                        self.endpoint_uri, request_data, **self.get_request_kwargs()
+                    )
+                except tuple(self.exception_retry_configuration.errors):
+                    if i < self.exception_retry_configuration.retries - 1:
+                        await asyncio.sleep(
+                            self.exception_retry_configuration.backoff_factor * 2**i
+                        )
+                        continue
+                    else:
+                        raise
+            return None
+        else:
+            return await self._request_session_manager.async_make_post_request(
+                self.endpoint_uri, request_data, **self.get_request_kwargs()
+            )
+
+    @async_handle_request_caching
+    async def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
+        self.logger.debug(
+            f"Making request HTTP. URI: {self.endpoint_uri}, Method: {method}"
+        )
+        request_data = self.encode_rpc_request(method, params)
+        raw_response = await self._make_request(method, request_data)
+        response = self.decode_rpc_response(raw_response)
+        self.logger.debug(
+            f"Getting response HTTP. URI: {self.endpoint_uri}, "
+            f"Method: {method}, Response: {response}"
+        )
+        return response
+
+    async def make_batch_request(
+        self, batch_requests: List[Tuple[RPCEndpoint, Any]]
+    ) -> List[RPCResponse]:
+        self.logger.debug(f"Making batch request HTTP - uri: `{self.endpoint_uri}`")
+        request_data = self.encode_batch_rpc_request(batch_requests)
+        raw_response = await self._request_session_manager.async_make_post_request(
+            self.endpoint_uri, request_data, **self.get_request_kwargs()
+        )
+        self.logger.debug("Received batch response HTTP.")
+        responses_list = cast(List[RPCResponse], self.decode_rpc_response(raw_response))
+        return sort_batch_response_by_response_ids(responses_list)