web3 7.0.0b5__py3-none-any.whl → 7.0.0b7__py3-none-any.whl

web3/providers/ipc.py

@@ -14,8 +14,11 @@ from types import (
 )
 from typing import (
     Any,
+    List,
+    Tuple,
     Type,
     Union,
+    cast,
 )
 
 from web3._utils.threads import (
@@ -26,6 +29,9 @@ from web3.types import (
     RPCResponse,
 )
 
+from .._utils.batching import (
+    sort_batch_response_by_response_ids,
+)
 from ..exceptions import (
     Web3TypeError,
     Web3ValueError,
@@ -135,7 +141,6 @@ class IPCProvider(JSONBaseProvider):
         self,
         ipc_path: Union[str, Path] = None,
         timeout: int = 30,
-        *args: Any,
         **kwargs: Any,
     ) -> None:
         if ipc_path is None:
@@ -148,17 +153,12 @@ class IPCProvider(JSONBaseProvider):
         self.timeout = timeout
         self._lock = threading.Lock()
         self._socket = PersistantSocket(self.ipc_path)
-        super().__init__()
+        super().__init__(**kwargs)
 
     def __str__(self) -> str:
         return f"<{self.__class__.__name__} {self.ipc_path}>"
 
-    def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
-        self.logger.debug(
-            f"Making request IPC. Path: {self.ipc_path}, Method: {method}"
-        )
-        request = self.encode_rpc_request(method, params)
-
+    def _make_request(self, request: bytes) -> RPCResponse:
         with self._lock, self._socket as sock:
             try:
                 sock.sendall(request)
@@ -189,6 +189,21 @@ class IPCProvider(JSONBaseProvider):
                         timeout.sleep(0)
                         continue
 
+    def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
+        self.logger.debug(
+            f"Making request IPC. Path: {self.ipc_path}, Method: {method}"
+        )
+        request = self.encode_rpc_request(method, params)
+        return self._make_request(request)
+
+    def make_batch_request(
+        self, requests: List[Tuple[RPCEndpoint, Any]]
+    ) -> List[RPCResponse]:
+        self.logger.debug(f"Making batch request IPC. Path: {self.ipc_path}")
+        request_data = self.encode_batch_rpc_request(requests)
+        response = cast(List[RPCResponse], self._make_request(request_data))
+        return sort_batch_response_by_response_ids(response)
+
 
 # A valid JSON RPC response can only end in } or ] http://www.jsonrpc.org/specification
 def has_valid_json_rpc_ending(raw_response: bytes) -> bool:

web3/providers/legacy_websocket.py

@@ -10,9 +10,12 @@ from types import (
 )
 from typing import (
     Any,
+    List,
     Optional,
+    Tuple,
     Type,
     Union,
+    cast,
 )
 
 from eth_typing import (
@@ -25,6 +28,12 @@ from websockets.legacy.client import (
     WebSocketClientProtocol,
 )
 
+from web3._utils.batching import (
+    sort_batch_response_by_response_ids,
+)
+from web3._utils.caching import (
+    handle_request_caching,
+)
 from web3.exceptions import (
     Web3ValidationError,
 )
@@ -91,6 +100,7 @@ class LegacyWebSocketProvider(JSONBaseProvider):
         endpoint_uri: Optional[Union[URI, str]] = None,
         websocket_kwargs: Optional[Any] = None,
         websocket_timeout: int = DEFAULT_WEBSOCKET_TIMEOUT,
+        **kwargs: Any,
     ) -> None:
         self.endpoint_uri = URI(endpoint_uri)
         self.websocket_timeout = websocket_timeout
@@ -110,7 +120,7 @@ class LegacyWebSocketProvider(JSONBaseProvider):
                     f"in websocket_kwargs, found: {found_restricted_keys}"
                 )
         self.conn = PersistentWebSocket(self.endpoint_uri, websocket_kwargs)
-        super().__init__()
+        super().__init__(**kwargs)
 
     def __str__(self) -> str:
         return f"WS connection {self.endpoint_uri}"
@@ -124,6 +134,7 @@ class LegacyWebSocketProvider(JSONBaseProvider):
                 await asyncio.wait_for(conn.recv(), timeout=self.websocket_timeout)
             )
 
+    @handle_request_caching
     def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
         self.logger.debug(
             f"Making request WebSocket. URI: {self.endpoint_uri}, " f"Method: {method}"
@@ -133,3 +144,17 @@ class LegacyWebSocketProvider(JSONBaseProvider):
             self.coro_make_request(request_data), LegacyWebSocketProvider._loop
         )
         return future.result()
+
+    def make_batch_request(
+        self, requests: List[Tuple[RPCEndpoint, Any]]
+    ) -> List[RPCResponse]:
+        self.logger.debug(
+            f"Making batch request WebSocket. URI: {self.endpoint_uri}, "
+            f"Methods: {requests}"
+        )
+        request_data = self.encode_batch_rpc_request(requests)
+        future = asyncio.run_coroutine_threadsafe(
+            self.coro_make_request(request_data), LegacyWebSocketProvider._loop
+        )
+        response = cast(List[RPCResponse], future.result())
+        return sort_batch_response_by_response_ids(response)

web3/providers/persistent/__init__.py

@@ -13,3 +13,10 @@ from .async_ipc import (
 from .websocket import (
     WebSocketProvider,
 )
+
+__all__ = [
+    "PersistentConnectionProvider",
+    "PersistentConnection",
+    "AsyncIPCProvider",
+    "WebSocketProvider",
+]

web3/providers/persistent/async_ipc.py

@@ -11,9 +11,11 @@ from pathlib import (
 import sys
 from typing import (
     Any,
+    List,
     Optional,
     Tuple,
     Union,
+    cast,
 )
 
 from eth_utils import (
@@ -28,6 +30,10 @@ from web3.types import (
 from . import (
     PersistentConnectionProvider,
 )
+from ..._utils.batching import (
+    BATCH_REQUEST_ID,
+    sort_batch_response_by_response_ids,
+)
 from ..._utils.caching import (
     async_handle_request_caching,
 )
@@ -59,11 +65,12 @@ class AsyncIPCProvider(PersistentConnectionProvider):
 
     _reader: Optional[asyncio.StreamReader] = None
     _writer: Optional[asyncio.StreamWriter] = None
+    _decoder: json.JSONDecoder = json.JSONDecoder()
+    _raw_message: str = ""
 
     def __init__(
         self,
         ipc_path: Optional[Union[str, Path]] = None,
-        max_connection_retries: int = 5,
         # `PersistentConnectionProvider` kwargs can be passed through
         **kwargs: Any,
     ) -> None:
@@ -74,7 +81,6 @@ class AsyncIPCProvider(PersistentConnectionProvider):
         else:
             raise Web3TypeError("ipc_path must be of type string or pathlib.Path")
 
-        self._max_connection_retries = max_connection_retries
         super().__init__(**kwargs)
 
     def __str__(self) -> str:
@@ -99,48 +105,16 @@ class AsyncIPCProvider(PersistentConnectionProvider):
                 )
             return False
 
-    async def connect(self) -> None:
-        _connection_attempts = 0
-        _backoff_rate_change = 1.75
-        _backoff_time = 1.75
-
-        while _connection_attempts != self._max_connection_retries:
-            try:
-                _connection_attempts += 1
-                self._reader, self._writer = await async_get_ipc_socket(self.ipc_path)
-                self._message_listener_task = asyncio.create_task(
-                    self._message_listener()
-                )
-                break
-            except OSError as e:
-                if _connection_attempts == self._max_connection_retries:
-                    raise ProviderConnectionError(
-                        f"Could not connect to: {self.ipc_path}. "
-                        f"Retries exceeded max of {self._max_connection_retries}."
-                    ) from e
-                self.logger.info(
-                    f"Could not connect to: {self.ipc_path}. Retrying in "
-                    f"{round(_backoff_time, 1)} seconds.",
-                    exc_info=True,
-                )
-                await asyncio.sleep(_backoff_time)
-                _backoff_time *= _backoff_rate_change
+    async def _provider_specific_connect(self) -> None:
+        self._reader, self._writer = await async_get_ipc_socket(self.ipc_path)
 
-    async def disconnect(self) -> None:
+    async def _provider_specific_disconnect(self) -> None:
         if self._writer and not self._writer.is_closing():
             self._writer.close()
             await self._writer.wait_closed()
             self._writer = None
-            self.logger.debug(f'Successfully disconnected from : "{self.ipc_path}')
-
-        try:
-            self._message_listener_task.cancel()
-            await self._message_listener_task
+        if self._reader:
             self._reader = None
-        except (asyncio.CancelledError, StopAsyncIteration):
-            pass
-
-        self._request_processor.clear_caches()
 
     async def _reset_socket(self) -> None:
         self._writer.close()
@@ -149,13 +123,12 @@ class AsyncIPCProvider(PersistentConnectionProvider):
 
     @async_handle_request_caching
     async def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
-        request_data = self.encode_rpc_request(method, params)
-
         if self._writer is None:
             raise ProviderConnectionError(
                 "Connection to ipc socket has not been initiated for the provider."
             )
 
+        request_data = self.encode_rpc_request(method, params)
         try:
             self._writer.write(request_data)
             await self._writer.drain()
@@ -172,43 +145,54 @@ class AsyncIPCProvider(PersistentConnectionProvider):
 
         return response
 
-    async def _message_listener(self) -> None:
-        self.logger.info(
-            "IPC socket listener background task started. Storing all messages in "
-            "appropriate request processor queues / caches to be processed."
+    async def make_batch_request(
+        self, requests: List[Tuple[RPCEndpoint, Any]]
+    ) -> List[RPCResponse]:
+        if self._writer is None:
+            raise ProviderConnectionError(
+                "Connection to ipc socket has not been initiated for the provider."
+            )
+
+        request_data = self.encode_batch_rpc_request(requests)
+        try:
+            self._writer.write(request_data)
+            await self._writer.drain()
+        except OSError as e:
+            # Broken pipe
+            if e.errno == errno.EPIPE:
+                # one extra attempt, then give up
+                await self._reset_socket()
+                self._writer.write(request_data)
+                await self._writer.drain()
+
+        response = cast(
+            List[RPCResponse], await self._get_response_for_request_id(BATCH_REQUEST_ID)
         )
-        raw_message = ""
-        decoder = json.JSONDecoder()
+        return response
 
-        while True:
-            # the use of sleep(0) seems to be the most efficient way to yield control
-            # back to the event loop to share the loop with other tasks.
-            await asyncio.sleep(0)
+    async def _provider_specific_message_listener(self) -> None:
+        self._raw_message += to_text(await self._reader.read(4096)).lstrip()
 
+        while self._raw_message:
             try:
-                raw_message += to_text(await self._reader.read(4096)).lstrip()
-
-                while raw_message:
-                    try:
-                        response, pos = decoder.raw_decode(raw_message)
-                    except JSONDecodeError:
-                        break
-
-                    is_subscription = response.get("method") == "eth_subscription"
-                    await self._request_processor.cache_raw_response(
-                        response, subscription=is_subscription
-                    )
-                    raw_message = raw_message[pos:].lstrip()
-            except Exception as e:
-                if not self.silence_listener_task_exceptions:
-                    loop = asyncio.get_event_loop()
-                    for task in asyncio.all_tasks(loop=loop):
-                        task.cancel()
-                    raise e
-
-                self.logger.error(
-                    "Exception caught in listener, error logging and keeping listener "
-                    f"background task alive.\n    error={e}"
-                )
-                # if only error logging, reset the ``raw_message`` buffer and continue
-                raw_message = ""
+                response, pos = self._decoder.raw_decode(self._raw_message)
+            except JSONDecodeError:
+                break
+
+            if isinstance(response, list):
+                response = sort_batch_response_by_response_ids(response)
+
+            is_subscription = (
+                response.get("method") == "eth_subscription"
+                if not isinstance(response, list)
+                else False
+            )
+            await self._request_processor.cache_raw_response(
+                response, subscription=is_subscription
+            )
+            self._raw_message = self._raw_message[pos:].lstrip()
+
+    def _error_log_listener_task_exception(self, e: Exception) -> None:
+        super()._error_log_listener_task_exception(e)
+        # reset the raw message buffer on exception when error logging
+        self._raw_message = ""

web3/providers/persistent/persistent.py

@@ -4,13 +4,23 @@ from abc import (
 import asyncio
 import logging
 from typing import (
+    Any,
+    List,
     Optional,
+    Union,
+)
+
+from websockets import (
+    ConnectionClosed,
+    WebSocketException,
 )
 
 from web3._utils.caching import (
     generate_cache_key,
 )
 from web3.exceptions import (
+    ProviderConnectionError,
+    TaskNotRunning,
     TimeExhausted,
 )
 from web3.providers.async_base import (
@@ -35,19 +45,24 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
     _message_listener_task: Optional["asyncio.Task[None]"] = None
     _listen_event: asyncio.Event = asyncio.Event()
 
+    _batch_request_counter: Optional[int] = None
+
     def __init__(
         self,
         request_timeout: float = DEFAULT_PERSISTENT_CONNECTION_TIMEOUT,
         subscription_response_queue_size: int = 500,
         silence_listener_task_exceptions: bool = False,
+        max_connection_retries: int = 5,
+        **kwargs: Any,
     ) -> None:
-        super().__init__()
+        super().__init__(**kwargs)
         self._request_processor = RequestProcessor(
             self,
             subscription_response_queue_size=subscription_response_queue_size,
         )
         self.request_timeout = request_timeout
         self.silence_listener_task_exceptions = silence_listener_task_exceptions
+        self._max_connection_retries = max_connection_retries
 
     def get_endpoint_uri_or_ipc_path(self) -> str:
         if hasattr(self, "endpoint_uri"):
@@ -61,16 +76,124 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
             )
 
     async def connect(self) -> None:
-        raise NotImplementedError("Must be implemented by subclasses")
+        _connection_attempts = 0
+        _backoff_rate_change = 1.75
+        _backoff_time = 1.75
+
+        while _connection_attempts != self._max_connection_retries:
+            try:
+                _connection_attempts += 1
+                self.logger.info(
+                    f"Connecting to: {self.get_endpoint_uri_or_ipc_path()}"
+                )
+                await self._provider_specific_connect()
+                self._message_listener_task = asyncio.create_task(
+                    self._message_listener()
+                )
+                self._message_listener_task.add_done_callback(
+                    self._message_listener_callback
+                )
+                self.logger.info(
+                    f"Successfully connected to: {self.get_endpoint_uri_or_ipc_path()}"
+                )
+                break
+            except (WebSocketException, OSError) as e:
+                if _connection_attempts == self._max_connection_retries:
+                    raise ProviderConnectionError(
+                        f"Could not connect to: {self.get_endpoint_uri_or_ipc_path()}. "
+                        f"Retries exceeded max of {self._max_connection_retries}."
+                    ) from e
+                self.logger.info(
+                    f"Could not connect to: {self.get_endpoint_uri_or_ipc_path()}. "
+                    f"Retrying in {round(_backoff_time, 1)} seconds.",
+                    exc_info=True,
+                )
+                await asyncio.sleep(_backoff_time)
+                _backoff_time *= _backoff_rate_change
 
     async def disconnect(self) -> None:
+        try:
+            if self._message_listener_task:
+                self._message_listener_task.cancel()
+                await self._message_listener_task
+        except (asyncio.CancelledError, StopAsyncIteration, ConnectionClosed):
+            pass
+        finally:
+            self._message_listener_task = None
+            self.logger.info("Message listener background task successfully shut down.")
+
+        await self._provider_specific_disconnect()
+        self._request_processor.clear_caches()
+        self.logger.info(
+            f"Successfully disconnected from: {self.get_endpoint_uri_or_ipc_path()}"
+        )
+
+    # -- private methods -- #
+
+    async def _provider_specific_connect(self) -> None:
         raise NotImplementedError("Must be implemented by subclasses")
 
-    async def _message_listener(self) -> None:
+    async def _provider_specific_disconnect(self) -> None:
         raise NotImplementedError("Must be implemented by subclasses")
 
+    async def _provider_specific_message_listener(self) -> None:
+        raise NotImplementedError("Must be implemented by subclasses")
+
+    def _message_listener_callback(
+        self, message_listener_task: "asyncio.Task[None]"
+    ) -> None:
+        # Puts a `TaskNotRunning` in the queue to signal the end of the listener task
+        # to any running subscription streams that are awaiting a response.
+        self._request_processor._subscription_response_queue.put_nowait(
+            TaskNotRunning(message_listener_task)
+        )
+
+    async def _message_listener(self) -> None:
+        self.logger.info(
+            f"{self.__class__.__qualname__} listener background task started. Storing "
+            "all messages in appropriate request processor queues / caches to be "
+            "processed."
+        )
+        while True:
+            # the use of sleep(0) seems to be the most efficient way to yield control
+            # back to the event loop to share the loop with other tasks.
+            await asyncio.sleep(0)
+            try:
+                await self._provider_specific_message_listener()
+            except Exception as e:
+                if not self.silence_listener_task_exceptions:
+                    raise e
+                else:
+                    self._error_log_listener_task_exception(e)
+
+    def _error_log_listener_task_exception(self, e: Exception) -> None:
+        """
+        When silencing listener task exceptions, this method is used to log the
+        exception and keep the listener task alive. Override this method to fine-tune
+        error logging behavior for the implementation class.
+        """
+        self.logger.error(
+            "Exception caught in listener, error logging and keeping "
+            "listener background task alive."
+            f"\n    error={e.__class__.__name__}: {e}"
+        )
+
+    def _handle_listener_task_exceptions(self) -> None:
+        """
+        Should be called every time a `PersistentConnectionProvider` is polling for
+        messages in the main loop. If the message listener task has completed and an
+        exception was recorded, raise the exception in the main loop.
+        """
+        msg_listener_task = getattr(self, "_message_listener_task", None)
+        if (
+            msg_listener_task
+            and msg_listener_task.done()
+            and msg_listener_task.exception()
+        ):
+            raise msg_listener_task.exception()
+
     async def _get_response_for_request_id(
-        self, request_id: RPCId, timeout: Optional[float] = None
+        self, request_id: Union[RPCId, List[RPCId]], timeout: Optional[float] = None
     ) -> RPCResponse:
         if timeout is None:
             timeout = self.request_timeout
@@ -79,10 +202,9 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
             request_cache_key = generate_cache_key(request_id)
 
             while True:
-                # sleep(0) here seems to be the most efficient way to yield control
-                # back to the event loop while waiting for the response to be in the
-                # queue.
-                await asyncio.sleep(0)
+                # check if an exception was recorded in the listener task and raise it
+                # in the main loop if so
+                self._handle_listener_task_exceptions()
 
                 if request_cache_key in self._request_processor._request_response_cache:
                     self.logger.debug(
@@ -92,11 +214,13 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
                         cache_key=request_cache_key,
                     )
                     return popped_response
+                else:
+                    await asyncio.sleep(0)
 
         try:
             # Add the request timeout around the while loop that checks the request
-            # cache and tried to recv(). If the request is neither in the cache, nor
-            # received within the request_timeout, raise ``TimeExhausted``.
+            # cache. If the request is not in the cache within the request_timeout,
+            # raise ``TimeExhausted``.
             return await asyncio.wait_for(_match_response_id_to_request_id(), timeout)
         except asyncio.TimeoutError:
             raise TimeExhausted(