web3 7.0.0b4__py3-none-any.whl → 7.0.0b6__py3-none-any.whl

web3/providers/legacy_websocket.py

@@ -10,9 +10,12 @@ from types import (
 )
 from typing import (
     Any,
+    List,
     Optional,
+    Tuple,
     Type,
     Union,
+    cast,
 )
 
 from eth_typing import (
@@ -25,6 +28,12 @@ from websockets.legacy.client import (
     WebSocketClientProtocol,
 )
 
+from web3._utils.batching import (
+    sort_batch_response_by_response_ids,
+)
+from web3._utils.caching import (
+    handle_request_caching,
+)
 from web3.exceptions import (
     Web3ValidationError,
 )
@@ -91,6 +100,7 @@ class LegacyWebSocketProvider(JSONBaseProvider):
         endpoint_uri: Optional[Union[URI, str]] = None,
         websocket_kwargs: Optional[Any] = None,
         websocket_timeout: int = DEFAULT_WEBSOCKET_TIMEOUT,
+        **kwargs: Any,
     ) -> None:
         self.endpoint_uri = URI(endpoint_uri)
         self.websocket_timeout = websocket_timeout
@@ -110,7 +120,7 @@ class LegacyWebSocketProvider(JSONBaseProvider):
                     f"in websocket_kwargs, found: {found_restricted_keys}"
                 )
         self.conn = PersistentWebSocket(self.endpoint_uri, websocket_kwargs)
-        super().__init__()
+        super().__init__(**kwargs)
 
     def __str__(self) -> str:
         return f"WS connection {self.endpoint_uri}"
@@ -124,6 +134,7 @@ class LegacyWebSocketProvider(JSONBaseProvider):
                 await asyncio.wait_for(conn.recv(), timeout=self.websocket_timeout)
             )
 
+    @handle_request_caching
     def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
         self.logger.debug(
             f"Making request WebSocket. URI: {self.endpoint_uri}, " f"Method: {method}"
@@ -133,3 +144,17 @@ class LegacyWebSocketProvider(JSONBaseProvider):
             self.coro_make_request(request_data), LegacyWebSocketProvider._loop
         )
         return future.result()
+
+    def make_batch_request(
+        self, requests: List[Tuple[RPCEndpoint, Any]]
+    ) -> List[RPCResponse]:
+        self.logger.debug(
+            f"Making batch request WebSocket. URI: {self.endpoint_uri}, "
+            f"Methods: {requests}"
+        )
+        request_data = self.encode_batch_rpc_request(requests)
+        future = asyncio.run_coroutine_threadsafe(
+            self.coro_make_request(request_data), LegacyWebSocketProvider._loop
+        )
+        response = cast(List[RPCResponse], future.result())
+        return sort_batch_response_by_response_ids(response)

web3/providers/persistent/async_ipc.py

@@ -11,9 +11,11 @@ from pathlib import (
 import sys
 from typing import (
     Any,
+    List,
     Optional,
     Tuple,
     Union,
+    cast,
 )
 
 from eth_utils import (
@@ -28,6 +30,10 @@ from web3.types import (
 from . import (
     PersistentConnectionProvider,
 )
+from ..._utils.batching import (
+    BATCH_REQUEST_ID,
+    sort_batch_response_by_response_ids,
+)
 from ..._utils.caching import (
     async_handle_request_caching,
 )
@@ -59,11 +65,12 @@ class AsyncIPCProvider(PersistentConnectionProvider):
 
     _reader: Optional[asyncio.StreamReader] = None
     _writer: Optional[asyncio.StreamWriter] = None
+    _decoder: json.JSONDecoder = json.JSONDecoder()
+    _raw_message: str = ""
 
     def __init__(
         self,
         ipc_path: Optional[Union[str, Path]] = None,
-        max_connection_retries: int = 5,
         # `PersistentConnectionProvider` kwargs can be passed through
         **kwargs: Any,
     ) -> None:
@@ -74,7 +81,6 @@ class AsyncIPCProvider(PersistentConnectionProvider):
         else:
             raise Web3TypeError("ipc_path must be of type string or pathlib.Path")
 
-        self._max_connection_retries = max_connection_retries
         super().__init__(**kwargs)
 
     def __str__(self) -> str:
@@ -99,48 +105,16 @@ class AsyncIPCProvider(PersistentConnectionProvider):
                 )
             return False
 
-    async def connect(self) -> None:
-        _connection_attempts = 0
-        _backoff_rate_change = 1.75
-        _backoff_time = 1.75
-
-        while _connection_attempts != self._max_connection_retries:
-            try:
-                _connection_attempts += 1
-                self._reader, self._writer = await async_get_ipc_socket(self.ipc_path)
-                self._message_listener_task = asyncio.create_task(
-                    self._message_listener()
-                )
-                break
-            except OSError as e:
-                if _connection_attempts == self._max_connection_retries:
-                    raise ProviderConnectionError(
-                        f"Could not connect to: {self.ipc_path}. "
-                        f"Retries exceeded max of {self._max_connection_retries}."
-                    ) from e
-                self.logger.info(
-                    f"Could not connect to: {self.ipc_path}. Retrying in "
-                    f"{round(_backoff_time, 1)} seconds.",
-                    exc_info=True,
-                )
-                await asyncio.sleep(_backoff_time)
-                _backoff_time *= _backoff_rate_change
+    async def _provider_specific_connect(self) -> None:
+        self._reader, self._writer = await async_get_ipc_socket(self.ipc_path)
 
-    async def disconnect(self) -> None:
+    async def _provider_specific_disconnect(self) -> None:
         if self._writer and not self._writer.is_closing():
             self._writer.close()
             await self._writer.wait_closed()
             self._writer = None
-            self.logger.debug(f'Successfully disconnected from : "{self.ipc_path}')
-
-        try:
-            self._message_listener_task.cancel()
-            await self._message_listener_task
+        if self._reader:
             self._reader = None
-        except (asyncio.CancelledError, StopAsyncIteration):
-            pass
-
-        self._request_processor.clear_caches()
 
     async def _reset_socket(self) -> None:
         self._writer.close()
@@ -149,13 +123,12 @@ class AsyncIPCProvider(PersistentConnectionProvider):
 
     @async_handle_request_caching
     async def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
-        request_data = self.encode_rpc_request(method, params)
-
         if self._writer is None:
             raise ProviderConnectionError(
                 "Connection to ipc socket has not been initiated for the provider."
             )
 
+        request_data = self.encode_rpc_request(method, params)
         try:
             self._writer.write(request_data)
             await self._writer.drain()
@@ -172,43 +145,54 @@ class AsyncIPCProvider(PersistentConnectionProvider):
 
         return response
 
-    async def _message_listener(self) -> None:
-        self.logger.info(
-            "IPC socket listener background task started. Storing all messages in "
-            "appropriate request processor queues / caches to be processed."
+    async def make_batch_request(
+        self, requests: List[Tuple[RPCEndpoint, Any]]
+    ) -> List[RPCResponse]:
+        if self._writer is None:
+            raise ProviderConnectionError(
+                "Connection to ipc socket has not been initiated for the provider."
+            )
+
+        request_data = self.encode_batch_rpc_request(requests)
+        try:
+            self._writer.write(request_data)
+            await self._writer.drain()
+        except OSError as e:
+            # Broken pipe
+            if e.errno == errno.EPIPE:
+                # one extra attempt, then give up
+                await self._reset_socket()
+                self._writer.write(request_data)
+                await self._writer.drain()
+
+        response = cast(
+            List[RPCResponse], await self._get_response_for_request_id(BATCH_REQUEST_ID)
         )
-        raw_message = ""
-        decoder = json.JSONDecoder()
+        return response
 
-        while True:
-            # the use of sleep(0) seems to be the most efficient way to yield control
-            # back to the event loop to share the loop with other tasks.
-            await asyncio.sleep(0)
+    async def _provider_specific_message_listener(self) -> None:
+        self._raw_message += to_text(await self._reader.read(4096)).lstrip()
 
+        while self._raw_message:
             try:
-                raw_message += to_text(await self._reader.read(4096)).lstrip()
-
-                while raw_message:
-                    try:
-                        response, pos = decoder.raw_decode(raw_message)
-                    except JSONDecodeError:
-                        break
-
-                    is_subscription = response.get("method") == "eth_subscription"
-                    await self._request_processor.cache_raw_response(
-                        response, subscription=is_subscription
-                    )
-                    raw_message = raw_message[pos:].lstrip()
-            except Exception as e:
-                if not self.silence_listener_task_exceptions:
-                    loop = asyncio.get_event_loop()
-                    for task in asyncio.all_tasks(loop=loop):
-                        task.cancel()
-                    raise e
-
-                self.logger.error(
-                    "Exception caught in listener, error logging and keeping listener "
-                    f"background task alive.\n    error={e}"
-                )
-                # if only error logging, reset the ``raw_message`` buffer and continue
-                raw_message = ""
+                response, pos = self._decoder.raw_decode(self._raw_message)
+            except JSONDecodeError:
+                break
+
+            if isinstance(response, list):
+                response = sort_batch_response_by_response_ids(response)
+
+            is_subscription = (
+                response.get("method") == "eth_subscription"
+                if not isinstance(response, list)
+                else False
+            )
+            await self._request_processor.cache_raw_response(
+                response, subscription=is_subscription
+            )
+            self._raw_message = self._raw_message[pos:].lstrip()
+
+    def _error_log_listener_task_exception(self, e: Exception) -> None:
+        super()._error_log_listener_task_exception(e)
+        # reset the raw message buffer on exception when error logging
+        self._raw_message = ""

web3/providers/persistent/persistent.py

@@ -4,13 +4,23 @@ from abc import (
 import asyncio
 import logging
 from typing import (
+    Any,
+    List,
     Optional,
+    Union,
+)
+
+from websockets import (
+    ConnectionClosed,
+    WebSocketException,
 )
 
 from web3._utils.caching import (
     generate_cache_key,
 )
 from web3.exceptions import (
+    ProviderConnectionError,
+    TaskNotRunning,
     TimeExhausted,
 )
 from web3.providers.async_base import (
@@ -35,19 +45,24 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
     _message_listener_task: Optional["asyncio.Task[None]"] = None
     _listen_event: asyncio.Event = asyncio.Event()
 
+    _batch_request_counter: Optional[int] = None
+
     def __init__(
         self,
         request_timeout: float = DEFAULT_PERSISTENT_CONNECTION_TIMEOUT,
         subscription_response_queue_size: int = 500,
         silence_listener_task_exceptions: bool = False,
+        max_connection_retries: int = 5,
+        **kwargs: Any,
     ) -> None:
-        super().__init__()
+        super().__init__(**kwargs)
         self._request_processor = RequestProcessor(
             self,
             subscription_response_queue_size=subscription_response_queue_size,
         )
         self.request_timeout = request_timeout
         self.silence_listener_task_exceptions = silence_listener_task_exceptions
+        self._max_connection_retries = max_connection_retries
 
     def get_endpoint_uri_or_ipc_path(self) -> str:
         if hasattr(self, "endpoint_uri"):
@@ -61,16 +76,124 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
             )
 
     async def connect(self) -> None:
-        raise NotImplementedError("Must be implemented by subclasses")
+        _connection_attempts = 0
+        _backoff_rate_change = 1.75
+        _backoff_time = 1.75
+
+        while _connection_attempts != self._max_connection_retries:
+            try:
+                _connection_attempts += 1
+                self.logger.info(
+                    f"Connecting to: {self.get_endpoint_uri_or_ipc_path()}"
+                )
+                await self._provider_specific_connect()
+                self._message_listener_task = asyncio.create_task(
+                    self._message_listener()
+                )
+                self._message_listener_task.add_done_callback(
+                    self._message_listener_callback
+                )
+                self.logger.info(
+                    f"Successfully connected to: {self.get_endpoint_uri_or_ipc_path()}"
+                )
+                break
+            except (WebSocketException, OSError) as e:
+                if _connection_attempts == self._max_connection_retries:
+                    raise ProviderConnectionError(
+                        f"Could not connect to: {self.get_endpoint_uri_or_ipc_path()}. "
+                        f"Retries exceeded max of {self._max_connection_retries}."
+                    ) from e
+                self.logger.info(
+                    f"Could not connect to: {self.get_endpoint_uri_or_ipc_path()}. "
+                    f"Retrying in {round(_backoff_time, 1)} seconds.",
+                    exc_info=True,
+                )
+                await asyncio.sleep(_backoff_time)
+                _backoff_time *= _backoff_rate_change
 
     async def disconnect(self) -> None:
+        try:
+            if self._message_listener_task:
+                self._message_listener_task.cancel()
+                await self._message_listener_task
+        except (asyncio.CancelledError, StopAsyncIteration, ConnectionClosed):
+            pass
+        finally:
+            self._message_listener_task = None
+            self.logger.info("Message listener background task successfully shut down.")
+
+        await self._provider_specific_disconnect()
+        self._request_processor.clear_caches()
+        self.logger.info(
+            f"Successfully disconnected from: {self.get_endpoint_uri_or_ipc_path()}"
+        )
+
+    # -- private methods -- #
+
+    async def _provider_specific_connect(self) -> None:
         raise NotImplementedError("Must be implemented by subclasses")
 
-    async def _message_listener(self) -> None:
+    async def _provider_specific_disconnect(self) -> None:
         raise NotImplementedError("Must be implemented by subclasses")
 
+    async def _provider_specific_message_listener(self) -> None:
+        raise NotImplementedError("Must be implemented by subclasses")
+
+    def _message_listener_callback(
+        self, message_listener_task: "asyncio.Task[None]"
+    ) -> None:
+        # Puts a `TaskNotRunning` in the queue to signal the end of the listener task
+        # to any running subscription streams that are awaiting a response.
+        self._request_processor._subscription_response_queue.put_nowait(
+            TaskNotRunning(message_listener_task)
+        )
+
+    async def _message_listener(self) -> None:
+        self.logger.info(
+            f"{self.__class__.__qualname__} listener background task started. Storing "
+            "all messages in appropriate request processor queues / caches to be "
+            "processed."
+        )
+        while True:
+            # the use of sleep(0) seems to be the most efficient way to yield control
+            # back to the event loop to share the loop with other tasks.
+            await asyncio.sleep(0)
+            try:
+                await self._provider_specific_message_listener()
+            except Exception as e:
+                if not self.silence_listener_task_exceptions:
+                    raise e
+                else:
+                    self._error_log_listener_task_exception(e)
+
+    def _error_log_listener_task_exception(self, e: Exception) -> None:
+        """
+        When silencing listener task exceptions, this method is used to log the
+        exception and keep the listener task alive. Override this method to fine-tune
+        error logging behavior for the implementation class.
+        """
+        self.logger.error(
+            "Exception caught in listener, error logging and keeping "
+            "listener background task alive."
+            f"\n    error={e.__class__.__name__}: {e}"
+        )
+
+    def _handle_listener_task_exceptions(self) -> None:
+        """
+        Should be called every time a `PersistentConnectionProvider` is polling for
+        messages in the main loop. If the message listener task has completed and an
+        exception was recorded, raise the exception in the main loop.
+        """
+        msg_listener_task = getattr(self, "_message_listener_task", None)
+        if (
+            msg_listener_task
+            and msg_listener_task.done()
+            and msg_listener_task.exception()
+        ):
+            raise msg_listener_task.exception()
+
     async def _get_response_for_request_id(
-        self, request_id: RPCId, timeout: Optional[float] = None
+        self, request_id: Union[RPCId, List[RPCId]], timeout: Optional[float] = None
     ) -> RPCResponse:
         if timeout is None:
             timeout = self.request_timeout
@@ -79,10 +202,9 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
             request_cache_key = generate_cache_key(request_id)
 
             while True:
-                # sleep(0) here seems to be the most efficient way to yield control
-                # back to the event loop while waiting for the response to be in the
-                # queue.
-                await asyncio.sleep(0)
+                # check if an exception was recorded in the listener task and raise it
+                # in the main loop if so
+                self._handle_listener_task_exceptions()
 
                 if request_cache_key in self._request_processor._request_response_cache:
                     self.logger.debug(
@@ -92,11 +214,13 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
                         cache_key=request_cache_key,
                     )
                     return popped_response
+                else:
+                    await asyncio.sleep(0)
 
         try:
             # Add the request timeout around the while loop that checks the request
-            # cache and tried to recv(). If the request is neither in the cache, nor
-            # received within the request_timeout, raise ``TimeExhausted``.
+            # cache. If the request is not in the cache within the request_timeout,
+            # raise ``TimeExhausted``.
             return await asyncio.wait_for(_match_response_id_to_request_id(), timeout)
         except asyncio.TimeoutError:
             raise TimeExhausted(

web3/providers/persistent/request_processor.py

@@ -2,20 +2,32 @@ import asyncio
 from copy import (
     copy,
 )
+import sys
 from typing import (
     TYPE_CHECKING,
     Any,
     Callable,
     Dict,
+    Generic,
     Optional,
     Tuple,
+    TypeVar,
+    Union,
 )
 
+from eth_utils.toolz import (
+    compose,
+)
+
+from web3._utils.batching import (
+    BATCH_REQUEST_ID,
+)
 from web3._utils.caching import (
     RequestInformation,
     generate_cache_key,
 )
 from web3.exceptions import (
+    TaskNotRunning,
     Web3ValueError,
 )
 from web3.types import (
@@ -31,6 +43,34 @@ if TYPE_CHECKING:
         PersistentConnectionProvider,
     )
 
+T = TypeVar("T")
+
+# TODO: This is an ugly hack for python 3.8. Remove this after we drop support for it
+#  and use `asyncio.Queue[T]` type directly in the `TaskReliantQueue` class.
+if sys.version_info >= (3, 9):
+
+    class _TaskReliantQueue(asyncio.Queue[T], Generic[T]):
+        pass
+
+else:
+
+    class _TaskReliantQueue(asyncio.Queue, Generic[T]):  # type: ignore
+        pass
+
+
+class TaskReliantQueue(_TaskReliantQueue[T]):
+    """
+    A queue that relies on a task to be running to process items in the queue.
+    """
+
+    async def get(self) -> T:
+        item = await super().get()
+        if isinstance(item, Exception):
+            # if the item is an exception, raise it so the task can handle this case
+            # more gracefully
+            raise item
+        return item
+
 
 class RequestProcessor:
     _subscription_queue_synced_with_ws_stream: bool = False
@@ -44,9 +84,9 @@ class RequestProcessor:
 
         self._request_information_cache: SimpleCache = SimpleCache(500)
         self._request_response_cache: SimpleCache = SimpleCache(500)
-        self._subscription_response_queue: asyncio.Queue[RPCResponse] = asyncio.Queue(
-            maxsize=subscription_response_queue_size
-        )
+        self._subscription_response_queue: TaskReliantQueue[
+            Union[RPCResponse, TaskNotRunning]
+        ] = TaskReliantQueue(maxsize=subscription_response_queue_size)
 
     @property
     def active_subscriptions(self) -> Dict[str, Any]:
@@ -62,7 +102,11 @@ class RequestProcessor:
         self,
         method: RPCEndpoint,
         params: Any,
-        response_formatters: Tuple[Callable[..., Any], ...],
+        response_formatters: Tuple[
+            Union[Dict[str, Callable[..., Any]], Callable[..., Any]],
+            Callable[..., Any],
+            Callable[..., Any],
+        ],
     ) -> Optional[str]:
         cached_requests_key = generate_cache_key((method, params))
         if cached_requests_key in self._provider._request_cache._data:
@@ -76,12 +120,17 @@ class RequestProcessor:
                 )
                 return None
 
-        # copy the request counter and find the next request id without incrementing
-        # since this is done when / if the request is successfully sent
-        request_id = next(copy(self._provider.request_counter))
-        cache_key = generate_cache_key(request_id)
+        if self._provider._is_batching:
+            # the _batch_request_counter is set when entering the context manager
+            current_request_id = self._provider._batch_request_counter
+            self._provider._batch_request_counter += 1
+        else:
+            # copy the request counter and find the next request id without incrementing
+            # since this is done when / if the request is successfully sent
+            current_request_id = next(copy(self._provider.request_counter))
+        cache_key = generate_cache_key(current_request_id)
 
-        self._bump_cache_if_key_present(cache_key, request_id)
+        self._bump_cache_if_key_present(cache_key, current_request_id)
 
         request_info = RequestInformation(
             method,
@@ -89,7 +138,7 @@ class RequestProcessor:
             response_formatters,
         )
         self._provider.logger.debug(
-            f"Caching request info:\n    request_id={request_id},\n"
+            f"Caching request info:\n    request_id={current_request_id},\n"
             f"    cache_key={cache_key},\n    request_info={request_info.__dict__}"
         )
         self._request_information_cache.cache(
@@ -153,9 +202,8 @@ class RequestProcessor:
                 # i.e. subscription request information remains in the cache
                 self._request_information_cache.get_cache_entry(cache_key)
             )
-
         else:
-            # retrieve the request info from the cache using the request id
+            # retrieve the request info from the cache using the response id
             cache_key = generate_cache_key(response["id"])
             if response in self._provider._request_cache._data.values():
                 request_info = (
@@ -184,6 +232,33 @@ class RequestProcessor:
 
         return request_info
 
+    def append_result_formatter_for_request(
+        self, request_id: int, result_formatter: Callable[..., Any]
+    ) -> None:
+        cache_key = generate_cache_key(request_id)
+        cached_request_info_for_id: RequestInformation = (
+            self._request_information_cache.get_cache_entry(cache_key)
+        )
+        if cached_request_info_for_id is not None:
+            (
+                current_result_formatters,
+                error_formatters,
+                null_result_formatters,
+            ) = cached_request_info_for_id.response_formatters
+            cached_request_info_for_id.response_formatters = (
+                compose(
+                    result_formatter,
+                    current_result_formatters,
+                ),
+                error_formatters,
+                null_result_formatters,
+            )
+        else:
+            self._provider.logger.debug(
+                f"No cached request info for response id `{request_id}`. Cannot "
+                f"append response formatter for response."
+            )
+
     def append_middleware_response_processor(
         self,
         response: RPCResponse,
@@ -218,7 +293,7 @@ class RequestProcessor:
     ) -> None:
         if subscription:
             if self._subscription_response_queue.full():
-                self._provider.logger.info(
+                self._provider.logger.debug(
                     "Subscription queue is full. Waiting for provider to consume "
                     "messages before caching."
                 )
@@ -229,6 +304,15 @@ class RequestProcessor:
                 f"Caching subscription response:\n    response={raw_response}"
             )
             await self._subscription_response_queue.put(raw_response)
+        elif isinstance(raw_response, list):
+            # Since only one batch should be in the cache at all times, we use a
+            # constant cache key for the batch response.
+            cache_key = generate_cache_key(BATCH_REQUEST_ID)
+            self._provider.logger.debug(
+                f"Caching batch response:\n    cache_key={cache_key},\n"
+                f"    response={raw_response}"
+            )
+            self._request_response_cache.cache(cache_key, raw_response)
         else:
             response_id = raw_response.get("id")
             cache_key = generate_cache_key(response_id)
@@ -289,6 +373,6 @@ class RequestProcessor:
         """Clear the request processor caches."""
         self._request_information_cache.clear()
         self._request_response_cache.clear()
-        self._subscription_response_queue = asyncio.Queue(
+        self._subscription_response_queue = TaskReliantQueue(
             maxsize=self._subscription_response_queue.maxsize
         )