web3 7.0.0b5__py3-none-any.whl → 7.0.0b6__py3-none-any.whl

web3/providers/persistent/persistent.py

@@ -4,13 +4,23 @@ from abc import (
 import asyncio
 import logging
 from typing import (
+    Any,
+    List,
     Optional,
+    Union,
+)
+
+from websockets import (
+    ConnectionClosed,
+    WebSocketException,
 )
 
 from web3._utils.caching import (
     generate_cache_key,
 )
 from web3.exceptions import (
+    ProviderConnectionError,
+    TaskNotRunning,
     TimeExhausted,
 )
 from web3.providers.async_base import (
@@ -35,19 +45,24 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
     _message_listener_task: Optional["asyncio.Task[None]"] = None
     _listen_event: asyncio.Event = asyncio.Event()
 
+    _batch_request_counter: Optional[int] = None
+
     def __init__(
         self,
         request_timeout: float = DEFAULT_PERSISTENT_CONNECTION_TIMEOUT,
         subscription_response_queue_size: int = 500,
         silence_listener_task_exceptions: bool = False,
+        max_connection_retries: int = 5,
+        **kwargs: Any,
     ) -> None:
-        super().__init__()
+        super().__init__(**kwargs)
         self._request_processor = RequestProcessor(
             self,
             subscription_response_queue_size=subscription_response_queue_size,
         )
         self.request_timeout = request_timeout
         self.silence_listener_task_exceptions = silence_listener_task_exceptions
+        self._max_connection_retries = max_connection_retries
 
     def get_endpoint_uri_or_ipc_path(self) -> str:
         if hasattr(self, "endpoint_uri"):
@@ -61,16 +76,124 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
             )
 
     async def connect(self) -> None:
-        raise NotImplementedError("Must be implemented by subclasses")
+        _connection_attempts = 0
+        _backoff_rate_change = 1.75
+        _backoff_time = 1.75
+
+        while _connection_attempts != self._max_connection_retries:
+            try:
+                _connection_attempts += 1
+                self.logger.info(
+                    f"Connecting to: {self.get_endpoint_uri_or_ipc_path()}"
+                )
+                await self._provider_specific_connect()
+                self._message_listener_task = asyncio.create_task(
+                    self._message_listener()
+                )
+                self._message_listener_task.add_done_callback(
+                    self._message_listener_callback
+                )
+                self.logger.info(
+                    f"Successfully connected to: {self.get_endpoint_uri_or_ipc_path()}"
+                )
+                break
+            except (WebSocketException, OSError) as e:
+                if _connection_attempts == self._max_connection_retries:
+                    raise ProviderConnectionError(
+                        f"Could not connect to: {self.get_endpoint_uri_or_ipc_path()}. "
+                        f"Retries exceeded max of {self._max_connection_retries}."
+                    ) from e
+                self.logger.info(
+                    f"Could not connect to: {self.get_endpoint_uri_or_ipc_path()}. "
+                    f"Retrying in {round(_backoff_time, 1)} seconds.",
+                    exc_info=True,
+                )
+                await asyncio.sleep(_backoff_time)
+                _backoff_time *= _backoff_rate_change
 
     async def disconnect(self) -> None:
+        try:
+            if self._message_listener_task:
+                self._message_listener_task.cancel()
+                await self._message_listener_task
+        except (asyncio.CancelledError, StopAsyncIteration, ConnectionClosed):
+            pass
+        finally:
+            self._message_listener_task = None
+            self.logger.info("Message listener background task successfully shut down.")
+
+        await self._provider_specific_disconnect()
+        self._request_processor.clear_caches()
+        self.logger.info(
+            f"Successfully disconnected from: {self.get_endpoint_uri_or_ipc_path()}"
+        )
+
+    # -- private methods -- #
+
+    async def _provider_specific_connect(self) -> None:
         raise NotImplementedError("Must be implemented by subclasses")
 
-    async def _message_listener(self) -> None:
+    async def _provider_specific_disconnect(self) -> None:
         raise NotImplementedError("Must be implemented by subclasses")
 
+    async def _provider_specific_message_listener(self) -> None:
+        raise NotImplementedError("Must be implemented by subclasses")
+
+    def _message_listener_callback(
+        self, message_listener_task: "asyncio.Task[None]"
+    ) -> None:
+        # Puts a `TaskNotRunning` in the queue to signal the end of the listener task
+        # to any running subscription streams that are awaiting a response.
+        self._request_processor._subscription_response_queue.put_nowait(
+            TaskNotRunning(message_listener_task)
+        )
+
+    async def _message_listener(self) -> None:
+        self.logger.info(
+            f"{self.__class__.__qualname__} listener background task started. Storing "
+            "all messages in appropriate request processor queues / caches to be "
+            "processed."
+        )
+        while True:
+            # the use of sleep(0) seems to be the most efficient way to yield control
+            # back to the event loop to share the loop with other tasks.
+            await asyncio.sleep(0)
+            try:
+                await self._provider_specific_message_listener()
+            except Exception as e:
+                if not self.silence_listener_task_exceptions:
+                    raise e
+                else:
+                    self._error_log_listener_task_exception(e)
+
+    def _error_log_listener_task_exception(self, e: Exception) -> None:
+        """
+        When silencing listener task exceptions, this method is used to log the
+        exception and keep the listener task alive. Override this method to fine-tune
+        error logging behavior for the implementation class.
+        """
+        self.logger.error(
+            "Exception caught in listener, error logging and keeping "
+            "listener background task alive."
+            f"\n    error={e.__class__.__name__}: {e}"
+        )
+
+    def _handle_listener_task_exceptions(self) -> None:
+        """
+        Should be called every time a `PersistentConnectionProvider` is polling for
+        messages in the main loop. If the message listener task has completed and an
+        exception was recorded, raise the exception in the main loop.
+        """
+        msg_listener_task = getattr(self, "_message_listener_task", None)
+        if (
+            msg_listener_task
+            and msg_listener_task.done()
+            and msg_listener_task.exception()
+        ):
+            raise msg_listener_task.exception()
+
     async def _get_response_for_request_id(
-        self, request_id: RPCId, timeout: Optional[float] = None
+        self, request_id: Union[RPCId, List[RPCId]], timeout: Optional[float] = None
     ) -> RPCResponse:
         if timeout is None:
             timeout = self.request_timeout
@@ -79,10 +202,9 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
             request_cache_key = generate_cache_key(request_id)
 
             while True:
-                # sleep(0) here seems to be the most efficient way to yield control
-                # back to the event loop while waiting for the response to be in the
-                # queue.
-                await asyncio.sleep(0)
+                # check if an exception was recorded in the listener task and raise it
+                # in the main loop if so
+                self._handle_listener_task_exceptions()
 
                 if request_cache_key in self._request_processor._request_response_cache:
                     self.logger.debug(
@@ -92,11 +214,13 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
                         cache_key=request_cache_key,
                     )
                     return popped_response
+                else:
+                    await asyncio.sleep(0)
 
         try:
             # Add the request timeout around the while loop that checks the request
-            # cache and tried to recv(). If the request is neither in the cache, nor
-            # received within the request_timeout, raise ``TimeExhausted``.
+            # cache. If the request is not in the cache within the request_timeout,
+            # raise ``TimeExhausted``.
             return await asyncio.wait_for(_match_response_id_to_request_id(), timeout)
         except asyncio.TimeoutError:
             raise TimeExhausted(

web3/providers/persistent/request_processor.py

@@ -2,20 +2,32 @@ import asyncio
 from copy import (
     copy,
 )
+import sys
 from typing import (
     TYPE_CHECKING,
     Any,
     Callable,
     Dict,
+    Generic,
     Optional,
     Tuple,
+    TypeVar,
+    Union,
 )
 
+from eth_utils.toolz import (
+    compose,
+)
+
+from web3._utils.batching import (
+    BATCH_REQUEST_ID,
+)
 from web3._utils.caching import (
     RequestInformation,
     generate_cache_key,
 )
 from web3.exceptions import (
+    TaskNotRunning,
     Web3ValueError,
 )
 from web3.types import (
@@ -31,6 +43,34 @@ if TYPE_CHECKING:
         PersistentConnectionProvider,
     )
 
+T = TypeVar("T")
+
+# TODO: This is an ugly hack for python 3.8. Remove this after we drop support for it
+#  and use `asyncio.Queue[T]` type directly in the `TaskReliantQueue` class.
+if sys.version_info >= (3, 9):
+
+    class _TaskReliantQueue(asyncio.Queue[T], Generic[T]):
+        pass
+
+else:
+
+    class _TaskReliantQueue(asyncio.Queue, Generic[T]):  # type: ignore
+        pass
+
+
+class TaskReliantQueue(_TaskReliantQueue[T]):
+    """
+    A queue that relies on a task to be running to process items in the queue.
+    """
+
+    async def get(self) -> T:
+        item = await super().get()
+        if isinstance(item, Exception):
+            # if the item is an exception, raise it so the task can handle this case
+            # more gracefully
+            raise item
+        return item
+
 
 class RequestProcessor:
     _subscription_queue_synced_with_ws_stream: bool = False
@@ -44,9 +84,9 @@ class RequestProcessor:
 
         self._request_information_cache: SimpleCache = SimpleCache(500)
         self._request_response_cache: SimpleCache = SimpleCache(500)
-        self._subscription_response_queue: asyncio.Queue[RPCResponse] = asyncio.Queue(
-            maxsize=subscription_response_queue_size
-        )
+        self._subscription_response_queue: TaskReliantQueue[
+            Union[RPCResponse, TaskNotRunning]
+        ] = TaskReliantQueue(maxsize=subscription_response_queue_size)
 
     @property
     def active_subscriptions(self) -> Dict[str, Any]:
@@ -62,7 +102,11 @@ class RequestProcessor:
         self,
         method: RPCEndpoint,
         params: Any,
-        response_formatters: Tuple[Callable[..., Any], ...],
+        response_formatters: Tuple[
+            Union[Dict[str, Callable[..., Any]], Callable[..., Any]],
+            Callable[..., Any],
+            Callable[..., Any],
+        ],
     ) -> Optional[str]:
         cached_requests_key = generate_cache_key((method, params))
         if cached_requests_key in self._provider._request_cache._data:
@@ -76,12 +120,17 @@ class RequestProcessor:
                 )
                 return None
 
-        # copy the request counter and find the next request id without incrementing
-        # since this is done when / if the request is successfully sent
-        request_id = next(copy(self._provider.request_counter))
-        cache_key = generate_cache_key(request_id)
+        if self._provider._is_batching:
+            # the _batch_request_counter is set when entering the context manager
+            current_request_id = self._provider._batch_request_counter
+            self._provider._batch_request_counter += 1
+        else:
+            # copy the request counter and find the next request id without incrementing
+            # since this is done when / if the request is successfully sent
+            current_request_id = next(copy(self._provider.request_counter))
+        cache_key = generate_cache_key(current_request_id)
 
-        self._bump_cache_if_key_present(cache_key, request_id)
+        self._bump_cache_if_key_present(cache_key, current_request_id)
 
         request_info = RequestInformation(
             method,
@@ -89,7 +138,7 @@ class RequestProcessor:
             response_formatters,
         )
         self._provider.logger.debug(
-            f"Caching request info:\n    request_id={request_id},\n"
+            f"Caching request info:\n    request_id={current_request_id},\n"
             f"    cache_key={cache_key},\n    request_info={request_info.__dict__}"
         )
         self._request_information_cache.cache(
@@ -153,9 +202,8 @@ class RequestProcessor:
                 # i.e. subscription request information remains in the cache
                 self._request_information_cache.get_cache_entry(cache_key)
             )
-
         else:
-            # retrieve the request info from the cache using the request id
+            # retrieve the request info from the cache using the response id
             cache_key = generate_cache_key(response["id"])
             if response in self._provider._request_cache._data.values():
                 request_info = (
@@ -184,6 +232,33 @@ class RequestProcessor:
 
         return request_info
 
+    def append_result_formatter_for_request(
+        self, request_id: int, result_formatter: Callable[..., Any]
+    ) -> None:
+        cache_key = generate_cache_key(request_id)
+        cached_request_info_for_id: RequestInformation = (
+            self._request_information_cache.get_cache_entry(cache_key)
+        )
+        if cached_request_info_for_id is not None:
+            (
+                current_result_formatters,
+                error_formatters,
+                null_result_formatters,
+            ) = cached_request_info_for_id.response_formatters
+            cached_request_info_for_id.response_formatters = (
+                compose(
+                    result_formatter,
+                    current_result_formatters,
+                ),
+                error_formatters,
+                null_result_formatters,
+            )
+        else:
+            self._provider.logger.debug(
+                f"No cached request info for response id `{request_id}`. Cannot "
+                f"append response formatter for response."
+            )
+
     def append_middleware_response_processor(
         self,
         response: RPCResponse,
@@ -218,7 +293,7 @@ class RequestProcessor:
     ) -> None:
         if subscription:
             if self._subscription_response_queue.full():
-                self._provider.logger.info(
+                self._provider.logger.debug(
                     "Subscription queue is full. Waiting for provider to consume "
                     "messages before caching."
                 )
@@ -229,6 +304,15 @@ class RequestProcessor:
                 f"Caching subscription response:\n    response={raw_response}"
             )
             await self._subscription_response_queue.put(raw_response)
+        elif isinstance(raw_response, list):
+            # Since only one batch should be in the cache at all times, we use a
+            # constant cache key for the batch response.
+            cache_key = generate_cache_key(BATCH_REQUEST_ID)
+            self._provider.logger.debug(
+                f"Caching batch response:\n    cache_key={cache_key},\n"
+                f"    response={raw_response}"
+            )
+            self._request_response_cache.cache(cache_key, raw_response)
         else:
             response_id = raw_response.get("id")
             cache_key = generate_cache_key(response_id)
@@ -289,6 +373,6 @@ class RequestProcessor:
         """Clear the request processor caches."""
         self._request_information_cache.clear()
         self._request_response_cache.clear()
-        self._subscription_response_queue = asyncio.Queue(
+        self._subscription_response_queue = TaskReliantQueue(
             maxsize=self._subscription_response_queue.maxsize
         )

web3/providers/persistent/websocket.py

@@ -5,8 +5,11 @@ import os
 from typing import (
     Any,
     Dict,
+    List,
     Optional,
+    Tuple,
     Union,
+    cast,
 )
 
 from eth_typing import (
@@ -25,6 +28,10 @@ from websockets.exceptions import (
     WebSocketException,
 )
 
+from web3._utils.batching import (
+    BATCH_REQUEST_ID,
+    sort_batch_response_by_response_ids,
+)
 from web3._utils.caching import (
     async_handle_request_caching,
 )
@@ -61,7 +68,6 @@ class WebSocketProvider(PersistentConnectionProvider):
     logger = logging.getLogger("web3.providers.WebSocketProvider")
     is_async: bool = True
 
-    _max_connection_retries: int = 5
     _ws: Optional[WebSocketClientProtocol] = None
 
     def __init__(
@@ -116,47 +122,13 @@ class WebSocketProvider(PersistentConnectionProvider):
                 ) from e
             return False
 
-    async def connect(self) -> None:
-        _connection_attempts = 0
-        _backoff_rate_change = 1.75
-        _backoff_time = 1.75
-
-        while _connection_attempts != self._max_connection_retries:
-            try:
-                _connection_attempts += 1
-                self._ws = await connect(self.endpoint_uri, **self.websocket_kwargs)
-                self._message_listener_task = asyncio.create_task(
-                    self._message_listener()
-                )
-                break
-            except WebSocketException as e:
-                if _connection_attempts == self._max_connection_retries:
-                    raise ProviderConnectionError(
-                        f"Could not connect to endpoint: {self.endpoint_uri}. "
-                        f"Retries exceeded max of {self._max_connection_retries}."
-                    ) from e
-                self.logger.info(
-                    f"Could not connect to endpoint: {self.endpoint_uri}. Retrying in "
-                    f"{round(_backoff_time, 1)} seconds.",
-                    exc_info=True,
-                )
-                await asyncio.sleep(_backoff_time)
-                _backoff_time *= _backoff_rate_change
+    async def _provider_specific_connect(self) -> None:
+        self._ws = await connect(self.endpoint_uri, **self.websocket_kwargs)
 
-    async def disconnect(self) -> None:
+    async def _provider_specific_disconnect(self) -> None:
         if self._ws is not None and not self._ws.closed:
             await self._ws.close()
             self._ws = None
-            self.logger.debug(
-                f'Successfully disconnected from endpoint: "{self.endpoint_uri}'
-            )
-
-        try:
-            self._message_listener_task.cancel()
-            await self._message_listener_task
-        except (asyncio.CancelledError, StopAsyncIteration):
-            pass
-        self._request_processor.clear_caches()
 
     @async_handle_request_caching
     async def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
@@ -176,34 +148,39 @@ class WebSocketProvider(PersistentConnectionProvider):
 
         return response
 
-    async def _message_listener(self) -> None:
-        self.logger.info(
-            "WebSocket listener background task started. Storing all messages in "
-            "appropriate request processor queues / caches to be processed."
+    async def make_batch_request(
+        self, requests: List[Tuple[RPCEndpoint, Any]]
+    ) -> List[RPCResponse]:
+        request_data = self.encode_batch_rpc_request(requests)
+
+        if self._ws is None:
+            raise ProviderConnectionError(
+                "Connection to websocket has not been initiated for the provider."
+            )
+
+        await asyncio.wait_for(
+            self._ws.send(request_data), timeout=self.request_timeout
+        )
+
+        response = cast(
+            List[RPCResponse],
+            await self._get_response_for_request_id(BATCH_REQUEST_ID),
         )
-        while True:
-            # the use of sleep(0) seems to be the most efficient way to yield control
-            # back to the event loop to share the loop with other tasks.
+        return response
+
+    async def _provider_specific_message_listener(self) -> None:
+        async for raw_message in self._ws:
             await asyncio.sleep(0)
 
-            try:
-                async for raw_message in self._ws:
-                    await asyncio.sleep(0)
-
-                    response = json.loads(raw_message)
-                    subscription = response.get("method") == "eth_subscription"
-                    await self._request_processor.cache_raw_response(
-                        response, subscription=subscription
-                    )
-            except Exception as e:
-                if not self.silence_listener_task_exceptions:
-                    loop = asyncio.get_event_loop()
-                    for task in asyncio.all_tasks(loop=loop):
-                        task.cancel()
-                    raise e
-
-                self.logger.error(
-                    "Exception caught in listener, error logging and keeping "
-                    "listener background task alive."
-                    f"\n    error={e.__class__.__name__}: {e}"
-                )
+            response = json.loads(raw_message)
+            if isinstance(response, list):
+                response = sort_batch_response_by_response_ids(response)
+
+            subscription = (
+                response.get("method") == "eth_subscription"
+                if not isinstance(response, list)
+                else False
+            )
+            await self._request_processor.cache_raw_response(
+                response, subscription=subscription
+            )

web3/providers/rpc/async_rpc.py

@@ -4,9 +4,11 @@ from typing import (
     Any,
     Dict,
     Iterable,
+    List,
     Optional,
     Tuple,
     Union,
+    cast,
 )
 
 from aiohttp import (
@@ -37,6 +39,9 @@ from web3.types import (
     RPCResponse,
 )
 
+from ..._utils.batching import (
+    sort_batch_response_by_response_ids,
+)
 from ..._utils.caching import (
     async_handle_request_caching,
 )
@@ -61,6 +66,7 @@ class AsyncHTTPProvider(AsyncJSONBaseProvider):
         exception_retry_configuration: Union[
             ExceptionRetryConfiguration, Empty
         ] = empty,
+        **kwargs: Any,
     ) -> None:
         if endpoint_uri is None:
             self.endpoint_uri = get_default_http_endpoint()
@@ -70,7 +76,7 @@ class AsyncHTTPProvider(AsyncJSONBaseProvider):
         self._request_kwargs = request_kwargs or {}
         self._exception_retry_configuration = exception_retry_configuration
 
-        super().__init__()
+        super().__init__(**kwargs)
 
     async def cache_async_session(self, session: ClientSession) -> ClientSession:
         return await _async_cache_and_return_session(self.endpoint_uri, session)
@@ -147,3 +153,15 @@ class AsyncHTTPProvider(AsyncJSONBaseProvider):
             f"Method: {method}, Response: {response}"
         )
         return response
+
+    async def make_batch_request(
+        self, batch_requests: List[Tuple[RPCEndpoint, Any]]
+    ) -> List[RPCResponse]:
+        self.logger.debug(f"Making batch request HTTP - uri: `{self.endpoint_uri}`")
+        request_data = self.encode_batch_rpc_request(batch_requests)
+        raw_response = await async_make_post_request(
+            self.endpoint_uri, request_data, **self.get_request_kwargs()
+        )
+        self.logger.debug("Received batch response HTTP.")
+        responses_list = cast(List[RPCResponse], self.decode_rpc_response(raw_response))
+        return sort_batch_response_by_response_ids(responses_list)

web3/providers/rpc/rpc.py

@@ -5,9 +5,11 @@ from typing import (
     Any,
     Dict,
     Iterable,
+    List,
     Optional,
     Tuple,
     Union,
+    cast,
 )
 
 from eth_typing import (
@@ -35,6 +37,9 @@ from web3.types import (
     RPCResponse,
 )
 
+from ..._utils.batching import (
+    sort_batch_response_by_response_ids,
+)
 from ..._utils.caching import (
     handle_request_caching,
 )
@@ -65,6 +70,7 @@ class HTTPProvider(JSONBaseProvider):
         exception_retry_configuration: Union[
             ExceptionRetryConfiguration, Empty
         ] = empty,
+        **kwargs: Any,
     ) -> None:
         if endpoint_uri is None:
             self.endpoint_uri = get_default_http_endpoint()
@@ -77,7 +83,7 @@ class HTTPProvider(JSONBaseProvider):
         if session:
             cache_and_return_session(self.endpoint_uri, session)
 
-        super().__init__()
+        super().__init__(**kwargs)
 
     def __str__(self) -> str:
         return f"RPC connection {self.endpoint_uri}"
@@ -155,3 +161,15 @@ class HTTPProvider(JSONBaseProvider):
             f"Method: {method}, Response: {response}"
         )
         return response
+
+    def make_batch_request(
+        self, batch_requests: List[Tuple[RPCEndpoint, Any]]
+    ) -> List[RPCResponse]:
+        self.logger.debug(f"Making batch request HTTP, uri: `{self.endpoint_uri}`")
+        request_data = self.encode_batch_rpc_request(batch_requests)
+        raw_response = make_post_request(
+            self.endpoint_uri, request_data, **self.get_request_kwargs()
+        )
+        self.logger.debug("Received batch response HTTP.")
+        responses_list = cast(List[RPCResponse], self.decode_rpc_response(raw_response))
+        return sort_batch_response_by_response_ids(responses_list)