web3 6.20.2__py3-none-any.whl → 7.0.0__py3-none-any.whl

web3/providers/{websocket/websocket.py → legacy_websocket.py}

@@ -10,9 +10,12 @@ from types import (
 )
 from typing import (
     Any,
+    List,
     Optional,
+    Tuple,
     Type,
     Union,
+    cast,
 )
 
 from eth_typing import (
@@ -25,6 +28,12 @@ from websockets.legacy.client import (
     WebSocketClientProtocol,
 )
 
+from web3._utils.batching import (
+    sort_batch_response_by_response_ids,
+)
+from web3._utils.caching import (
+    handle_request_caching,
+)
 from web3.exceptions import (
     Web3ValidationError,
 )
@@ -37,7 +46,7 @@ from web3.types import (
 )
 
 RESTRICTED_WEBSOCKET_KWARGS = {"uri", "loop"}
-DEFAULT_WEBSOCKET_TIMEOUT = 10
+DEFAULT_WEBSOCKET_TIMEOUT = 30
 
 
 def _start_event_loop(loop: asyncio.AbstractEventLoop) -> None:
@@ -82,8 +91,8 @@ class PersistentWebSocket:
             self.ws = None
 
 
-class WebsocketProvider(JSONBaseProvider):
-    logger = logging.getLogger("web3.providers.WebsocketProvider")
+class LegacyWebSocketProvider(JSONBaseProvider):
+    logger = logging.getLogger("web3.providers.WebSocketProvider")
     _loop = None
 
     def __init__(
@@ -91,13 +100,14 @@ class WebsocketProvider(JSONBaseProvider):
         endpoint_uri: Optional[Union[URI, str]] = None,
         websocket_kwargs: Optional[Any] = None,
         websocket_timeout: int = DEFAULT_WEBSOCKET_TIMEOUT,
+        **kwargs: Any,
     ) -> None:
         self.endpoint_uri = URI(endpoint_uri)
         self.websocket_timeout = websocket_timeout
         if self.endpoint_uri is None:
             self.endpoint_uri = get_default_endpoint()
-        if WebsocketProvider._loop is None:
-            WebsocketProvider._loop = _get_threaded_loop()
+        if LegacyWebSocketProvider._loop is None:
+            LegacyWebSocketProvider._loop = _get_threaded_loop()
         if websocket_kwargs is None:
             websocket_kwargs = {}
         else:
@@ -110,7 +120,7 @@ class WebsocketProvider(JSONBaseProvider):
                     f"in websocket_kwargs, found: {found_restricted_keys}"
                 )
         self.conn = PersistentWebSocket(self.endpoint_uri, websocket_kwargs)
-        super().__init__()
+        super().__init__(**kwargs)
 
     def __str__(self) -> str:
         return f"WS connection {self.endpoint_uri}"
@@ -124,12 +134,27 @@ class WebsocketProvider(JSONBaseProvider):
                 await asyncio.wait_for(conn.recv(), timeout=self.websocket_timeout)
             )
 
+    @handle_request_caching
     def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
         self.logger.debug(
             f"Making request WebSocket. URI: {self.endpoint_uri}, " f"Method: {method}"
         )
         request_data = self.encode_rpc_request(method, params)
         future = asyncio.run_coroutine_threadsafe(
-            self.coro_make_request(request_data), WebsocketProvider._loop
+            self.coro_make_request(request_data), LegacyWebSocketProvider._loop
         )
         return future.result()
+
+    def make_batch_request(
+        self, requests: List[Tuple[RPCEndpoint, Any]]
+    ) -> List[RPCResponse]:
+        self.logger.debug(
+            f"Making batch request WebSocket. URI: {self.endpoint_uri}, "
+            f"Methods: {requests}"
+        )
+        request_data = self.encode_batch_rpc_request(requests)
+        future = asyncio.run_coroutine_threadsafe(
+            self.coro_make_request(request_data), LegacyWebSocketProvider._loop
+        )
+        response = cast(List[RPCResponse], future.result())
+        return sort_batch_response_by_response_ids(response)

web3/providers/persistent/__init__.py

@@ -0,0 +1,22 @@
+from .persistent import (
+    PersistentConnectionProvider,
+)
+from .persistent_connection import (
+    PersistentConnection,
+)
+from .request_processor import (
+    RequestProcessor,
+)
+from .async_ipc import (
+    AsyncIPCProvider,
+)
+from .websocket import (
+    WebSocketProvider,
+)
+
+__all__ = [
+    "PersistentConnectionProvider",
+    "PersistentConnection",
+    "AsyncIPCProvider",
+    "WebSocketProvider",
+]

web3/providers/persistent/async_ipc.py

@@ -0,0 +1,153 @@
+import asyncio
+import errno
+import json
+from json import (
+    JSONDecodeError,
+)
+import logging
+from pathlib import (
+    Path,
+)
+import sys
+from typing import (
+    Any,
+    Optional,
+    Tuple,
+    Union,
+)
+
+from eth_utils import (
+    to_text,
+)
+
+from web3.types import (
+    RPCResponse,
+)
+
+from . import (
+    PersistentConnectionProvider,
+)
+from ...exceptions import (
+    ProviderConnectionError,
+    Web3TypeError,
+)
+from ..ipc import (
+    get_default_ipc_path,
+)
+
+
+async def async_get_ipc_socket(
+    ipc_path: str,
+) -> Tuple[asyncio.StreamReader, asyncio.StreamWriter]:
+    if sys.platform == "win32":
+        # On Windows named pipe is used. Simulate socket with it.
+        from web3._utils.windows import (
+            NamedPipe,
+        )
+
+        return NamedPipe(ipc_path)
+    else:
+        return await asyncio.open_unix_connection(ipc_path)
+
+
+class AsyncIPCProvider(PersistentConnectionProvider):
+    logger = logging.getLogger("web3.providers.AsyncIPCProvider")
+
+    _reader: Optional[asyncio.StreamReader] = None
+    _writer: Optional[asyncio.StreamWriter] = None
+    _decoder: json.JSONDecoder = json.JSONDecoder()
+    _raw_message: str = ""
+
+    def __init__(
+        self,
+        ipc_path: Optional[Union[str, Path]] = None,
+        # `PersistentConnectionProvider` kwargs can be passed through
+        **kwargs: Any,
+    ) -> None:
+        if ipc_path is None:
+            self.ipc_path = get_default_ipc_path()
+        elif isinstance(ipc_path, str) or isinstance(ipc_path, Path):
+            self.ipc_path = str(Path(ipc_path).expanduser().resolve())
+        else:
+            raise Web3TypeError("ipc_path must be of type string or pathlib.Path")
+
+        super().__init__(**kwargs)
+
+    def __str__(self) -> str:
+        return f"<{self.__class__.__name__} {self.ipc_path}>"
+
+    async def is_connected(self, show_traceback: bool = False) -> bool:
+        if not self._writer or not self._reader:
+            return False
+
+        try:
+            await self.make_request("web3_clientVersion", [])
+            return True
+        except (OSError, ProviderConnectionError) as e:
+            if show_traceback:
+                raise ProviderConnectionError(
+                    f"Problem connecting to provider with error: {type(e)}: {e}"
+                )
+            return False
+
+    async def socket_send(self, request_data: bytes) -> None:
+        if self._writer is None:
+            raise ProviderConnectionError(
+                "Connection to ipc socket has not been initiated for the provider."
+            )
+
+        return await asyncio.wait_for(
+            self._socket_send(request_data), timeout=self.request_timeout
+        )
+
+    async def socket_recv(self) -> RPCResponse:
+        while True:
+            # yield to the event loop to allow other tasks to run
+            await asyncio.sleep(0)
+
+            try:
+                response, pos = self._decoder.raw_decode(self._raw_message)
+                self._raw_message = self._raw_message[pos:].lstrip()
+                return response
+            except JSONDecodeError:
+                # read more data from the socket if the current raw message is
+                # incomplete
+                self._raw_message += to_text(await self._reader.read(4096)).lstrip()
+
+    # -- private methods -- #
+
+    async def _socket_send(self, request_data: bytes) -> None:
+        try:
+            self._writer.write(request_data)
+            await self._writer.drain()
+        except OSError as e:
+            # Broken pipe
+            if e.errno == errno.EPIPE:
+                # one extra attempt, then give up
+                await self._reset_socket()
+                self._writer.write(request_data)
+                await self._writer.drain()
+
+    async def _reset_socket(self) -> None:
+        self._writer.close()
+        await self._writer.wait_closed()
+        self._reader, self._writer = await async_get_ipc_socket(self.ipc_path)
+
+    async def _provider_specific_connect(self) -> None:
+        self._reader, self._writer = await async_get_ipc_socket(self.ipc_path)
+
+    async def _provider_specific_disconnect(self) -> None:
+        if self._writer and not self._writer.is_closing():
+            self._writer.close()
+            await self._writer.wait_closed()
+            self._writer = None
+        if self._reader:
+            self._reader = None
+
+    async def _provider_specific_socket_reader(self) -> RPCResponse:
+        return await self.socket_recv()
+
+    def _error_log_listener_task_exception(self, e: Exception) -> None:
+        super()._error_log_listener_task_exception(e)
+        # reset the raw message buffer on exception when error logging
+        self._raw_message = ""

web3/providers/{persistent.py → persistent/persistent.py}

@@ -1,68 +1,91 @@
 from abc import (
     ABC,
+    abstractmethod,
 )
 import asyncio
+import json
 import logging
 from typing import (
+    Any,
+    List,
     Optional,
+    Tuple,
+    Union,
+    cast,
 )
 
 from websockets import (
     ConnectionClosed,
-    ConnectionClosedOK,
-    WebSocketClientProtocol,
     WebSocketException,
 )
 
+from web3._utils.batching import (
+    BATCH_REQUEST_ID,
+    sort_batch_response_by_response_ids,
+)
 from web3._utils.caching import (
+    async_handle_request_caching,
     generate_cache_key,
 )
 from web3.exceptions import (
+    PersistentConnectionClosedOK,
     ProviderConnectionError,
     TaskNotRunning,
     TimeExhausted,
+    Web3AttributeError,
 )
 from web3.providers.async_base import (
     AsyncJSONBaseProvider,
 )
-from web3.providers.websocket.request_processor import (
+from web3.providers.persistent.request_processor import (
     RequestProcessor,
 )
 from web3.types import (
+    RPCEndpoint,
     RPCId,
     RPCResponse,
 )
 
-DEFAULT_PERSISTENT_CONNECTION_TIMEOUT = 50.0
+DEFAULT_PERSISTENT_CONNECTION_TIMEOUT = 30.0
 
 
 class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
     logger = logging.getLogger("web3.providers.PersistentConnectionProvider")
     has_persistent_connection = True
-    endpoint_uri: Optional[str] = None
-
-    _max_connection_retries: int = 5
 
-    _ws: Optional[WebSocketClientProtocol] = None
     _request_processor: RequestProcessor
     _message_listener_task: Optional["asyncio.Task[None]"] = None
     _listen_event: asyncio.Event = asyncio.Event()
 
+    _batch_request_counter: Optional[int] = None
+
     def __init__(
         self,
         request_timeout: float = DEFAULT_PERSISTENT_CONNECTION_TIMEOUT,
         subscription_response_queue_size: int = 500,
-        request_information_cache_size: int = 500,
         silence_listener_task_exceptions: bool = False,
+        max_connection_retries: int = 5,
+        **kwargs: Any,
     ) -> None:
-        super().__init__()
+        super().__init__(**kwargs)
         self._request_processor = RequestProcessor(
             self,
             subscription_response_queue_size=subscription_response_queue_size,
-            request_information_cache_size=request_information_cache_size,
         )
         self.request_timeout = request_timeout
         self.silence_listener_task_exceptions = silence_listener_task_exceptions
+        self._max_connection_retries = max_connection_retries
+
+    def get_endpoint_uri_or_ipc_path(self) -> str:
+        if hasattr(self, "endpoint_uri"):
+            return str(self.endpoint_uri)
+        elif hasattr(self, "ipc_path"):
+            return str(self.ipc_path)
+        else:
+            raise Web3AttributeError(
+                "`PersistentConnectionProvider` must have either `endpoint_uri` or "
+                "`ipc_path` attribute."
+            )
 
     async def connect(self) -> None:
         _connection_attempts = 0
@@ -72,7 +95,9 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
         while _connection_attempts != self._max_connection_retries:
             try:
                 _connection_attempts += 1
-                self.logger.info(f"Connecting to: {self.endpoint_uri}")
+                self.logger.info(
+                    f"Connecting to: {self.get_endpoint_uri_or_ipc_path()}"
+                )
                 await self._provider_specific_connect()
                 self._message_listener_task = asyncio.create_task(
                     self._message_listener()
@@ -80,16 +105,18 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
                 self._message_listener_task.add_done_callback(
                     self._message_listener_callback
                 )
-                self.logger.info(f"Successfully connected to: {self.endpoint_uri}")
+                self.logger.info(
+                    f"Successfully connected to: {self.get_endpoint_uri_or_ipc_path()}"
+                )
                 break
             except (WebSocketException, OSError) as e:
                 if _connection_attempts == self._max_connection_retries:
                     raise ProviderConnectionError(
-                        f"Could not connect to: {self.endpoint_uri}. "
+                        f"Could not connect to: {self.get_endpoint_uri_or_ipc_path()}. "
                         f"Retries exceeded max of {self._max_connection_retries}."
                     ) from e
                 self.logger.info(
-                    f"Could not connect to: {self.endpoint_uri}. "
+                    f"Could not connect to: {self.get_endpoint_uri_or_ipc_path()}. "
                     f"Retrying in {round(_backoff_time, 1)} seconds.",
                     exc_info=True,
                 )
@@ -109,7 +136,47 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
 
         await self._provider_specific_disconnect()
         self._request_processor.clear_caches()
-        self.logger.info(f"Successfully disconnected from: {self.endpoint_uri}")
+        self.logger.info(
+            f"Successfully disconnected from: {self.get_endpoint_uri_or_ipc_path()}"
+        )
+
+    @async_handle_request_caching
+    async def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
+        request_data = self.encode_rpc_request(method, params)
+        await self.socket_send(request_data)
+
+        current_request_id = json.loads(request_data)["id"]
+        response = await self._get_response_for_request_id(current_request_id)
+
+        return response
+
+    async def make_batch_request(
+        self, requests: List[Tuple[RPCEndpoint, Any]]
+    ) -> List[RPCResponse]:
+        request_data = self.encode_batch_rpc_request(requests)
+        await self.socket_send(request_data)
+
+        response = cast(
+            List[RPCResponse], await self._get_response_for_request_id(BATCH_REQUEST_ID)
+        )
+        return response
+
+    # -- abstract methods -- #
+
+    @abstractmethod
+    async def socket_send(self, request_data: bytes) -> None:
+        """
+        Send an encoded RPC request to the provider over the persistent connection.
+        """
+        raise NotImplementedError("Must be implemented by subclasses")
+
+    @abstractmethod
+    async def socket_recv(self) -> RPCResponse:
+        """
+        Receive, decode, and return an RPC response from the provider over the
+        persistent connection.
+        """
+        raise NotImplementedError("Must be implemented by subclasses")
 
     # -- private methods -- #
 
@@ -119,7 +186,7 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
     async def _provider_specific_disconnect(self) -> None:
         raise NotImplementedError("Must be implemented by subclasses")
 
-    async def _provider_specific_message_listener(self) -> None:
+    async def _provider_specific_socket_reader(self) -> RPCResponse:
         raise NotImplementedError("Must be implemented by subclasses")
 
     def _message_listener_callback(
@@ -141,14 +208,28 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
             # the use of sleep(0) seems to be the most efficient way to yield control
             # back to the event loop to share the loop with other tasks.
             await asyncio.sleep(0)
+
             try:
-                await self._provider_specific_message_listener()
-            except ConnectionClosedOK:
-                # RequestManager._ws_message_stream() will rethrow this exception and
-                # cause WebsocketConnection to stop the process_subscriptions() loop,
-                # or raise ConnectionClosedOK when recv() is called.
-                # see: https://github.com/ethereum/web3.py/pull/3424
-                raise
+                response = await self._provider_specific_socket_reader()
+
+                if isinstance(response, list):
+                    response = sort_batch_response_by_response_ids(response)
+
+                subscription = (
+                    response.get("method") == "eth_subscription"
+                    if not isinstance(response, list)
+                    else False
+                )
+                await self._request_processor.cache_raw_response(
+                    response, subscription=subscription
+                )
+            except PersistentConnectionClosedOK as e:
+                self.logger.info(
+                    "Message listener background task has ended gracefully: "
+                    f"{e.user_message}"
+                )
+                # trigger a return to end the listener task and initiate the callback fn
+                return
             except Exception as e:
                 if not self.silence_listener_task_exceptions:
                     raise e
@@ -182,7 +263,7 @@ class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
             raise msg_listener_task.exception()
 
     async def _get_response_for_request_id(
-        self, request_id: RPCId, timeout: Optional[float] = None
+        self, request_id: Union[RPCId, List[RPCId]], timeout: Optional[float] = None
     ) -> RPCResponse:
         if timeout is None:
             timeout = self.request_timeout

web3/providers/persistent/persistent_connection.py

@@ -0,0 +1,84 @@
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+)
+
+from web3.types import (
+    RPCEndpoint,
+    RPCResponse,
+)
+
+if TYPE_CHECKING:
+    from web3.main import (  # noqa: F401
+        AsyncWeb3,
+    )
+    from web3.manager import (  # noqa: F401
+        _AsyncPersistentMessageStream,
+    )
+
+
+class PersistentConnection:
+    """
+    A class that houses the public API for interacting with the persistent connection
+    via a `AsyncWeb3` instance instantiated with a `PersistentConnectionProvider` class.
+    """
+
+    def __init__(self, w3: "AsyncWeb3"):
+        self._manager = w3.manager
+
+    @property
+    def subscriptions(self) -> Dict[str, Any]:
+        """
+        Return the active subscriptions on the persistent connection.
+
+        :return: The active subscriptions on the persistent connection.
+        :rtype: Dict[str, Any]
+        """
+        return self._manager._request_processor.active_subscriptions
+
+    async def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
+        """
+        Make a request to the persistent connection and return the response. This method
+        does not process the response as it would when invoking a method via the
+        appropriate module on the `AsyncWeb3` instance,
+        e.g. `w3.eth.get_block("latest")`.
+
+        :param method: The RPC method, e.g. `eth_getBlockByNumber`.
+        :param params: The RPC method parameters, e.g. `["0x1337", False]`.
+
+        :return: The processed response from the persistent connection.
+        :rtype: RPCResponse
+        """
+        return await self._manager.socket_request(method, params)
+
+    async def send(self, method: RPCEndpoint, params: Any) -> None:
+        """
+        Send a raw, unprocessed message to the persistent connection.
+
+        :param method: The RPC method, e.g. `eth_getBlockByNumber`.
+        :param params: The RPC method parameters, e.g. `["0x1337", False]`.
+
+        :return: None
+        """
+        await self._manager.send(method, params)
+
+    async def recv(self) -> RPCResponse:
+        """
+        Receive the next unprocessed response for a request from the persistent
+        connection.
+
+        :return: The next unprocessed response for a request from the persistent
+                 connection.
+        :rtype: RPCResponse
+        """
+        return await self._manager.recv()
+
+    def process_subscriptions(self) -> "_AsyncPersistentMessageStream":
+        """
+        Asynchronous iterator that yields messages from the subscription message stream.
+
+        :return: The subscription message stream.
+        :rtype: _AsyncPersistentMessageStream
+        """
+        return self._manager._persistent_message_stream()