web3 7.0.0b1__py3-none-any.whl → 7.7.0__py3-none-any.whl

web3/providers/ipc.py

@@ -14,8 +14,11 @@ from types import (
 )
 from typing import (
     Any,
+    List,
+    Tuple,
     Type,
     Union,
+    cast,
 )
 
 from web3._utils.threads import (
@@ -26,6 +29,16 @@ from web3.types import (
     RPCResponse,
 )
 
+from .._utils.batching import (
+    sort_batch_response_by_response_ids,
+)
+from .._utils.caching import (
+    handle_request_caching,
+)
+from ..exceptions import (
+    Web3TypeError,
+    Web3ValueError,
+)
 from .base import (
     JSONBaseProvider,
 )
@@ -96,28 +109,29 @@ def get_default_ipc_path() -> str:
         return r"\\.\pipe\geth.ipc"
 
     else:
-        raise ValueError(
+        raise Web3ValueError(
             f"Unsupported platform '{sys.platform}'. Only darwin/linux/win32/"
             "freebsd are supported.  You must specify the ipc_path"
         )
 
 
 def get_dev_ipc_path() -> str:
-    if os.environ.get("WEB3_PROVIDER_URI", ""):
-        return os.environ.get("WEB3_PROVIDER_URI")
+    web3_provider_uri = os.environ.get("WEB3_PROVIDER_URI", "")
+    if web3_provider_uri and "geth.ipc" in web3_provider_uri:
+        return web3_provider_uri
 
-    elif sys.platform == "darwin":
-        tmpdir = os.environ.get("TMPDIR", "")
+    elif sys.platform == "darwin" or sys.platform.startswith("linux"):
+        tmpdir = os.environ.get("TMPDIR", "/tmp")
         return os.path.expanduser(os.path.join(tmpdir, "geth.ipc"))
 
-    elif sys.platform.startswith("linux") or sys.platform.startswith("freebsd"):
+    elif sys.platform.endswith("freebsd"):
         return os.path.expanduser(os.path.join("/tmp", "geth.ipc"))
 
     elif sys.platform == "win32":
         return r"\\.\pipe\geth.ipc"
 
     else:
-        raise ValueError(
+        raise Web3ValueError(
             f"Unsupported platform '{sys.platform}'. Only darwin/linux/win32/"
             "freebsd are supported.  You must specify the ipc_path"
         )
@@ -130,34 +144,28 @@ class IPCProvider(JSONBaseProvider):
     def __init__(
         self,
         ipc_path: Union[str, Path] = None,
-        timeout: int = 10,
-        *args: Any,
+        timeout: int = 30,
         **kwargs: Any,
     ) -> None:
+        super().__init__(**kwargs)
         if ipc_path is None:
             self.ipc_path = get_default_ipc_path()
         elif isinstance(ipc_path, str) or isinstance(ipc_path, Path):
             self.ipc_path = str(Path(ipc_path).expanduser().resolve())
         else:
-            raise TypeError("ipc_path must be of type string or pathlib.Path")
+            raise Web3TypeError("ipc_path must be of type string or pathlib.Path")
 
         self.timeout = timeout
         self._lock = threading.Lock()
         self._socket = PersistantSocket(self.ipc_path)
-        super().__init__()
 
     def __str__(self) -> str:
         return f"<{self.__class__.__name__} {self.ipc_path}>"
 
-    def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
-        self.logger.debug(
-            f"Making request IPC. Path: {self.ipc_path}, Method: {method}"
-        )
-        request = self.encode_rpc_request(method, params)
-
+    def _make_request(self, request: bytes) -> RPCResponse:
         with self._lock, self._socket as sock:
             try:
-                sock.sendall(request)
+                sock.sendall(request + b"\n")
             except BrokenPipeError:
                 # one extra attempt, then give up
                 sock = self._socket.reset()
@@ -185,6 +193,22 @@ class IPCProvider(JSONBaseProvider):
                         timeout.sleep(0)
                         continue
 
+    @handle_request_caching
+    def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
+        self.logger.debug(
+            f"Making request IPC. Path: {self.ipc_path}, Method: {method}"
+        )
+        request = self.encode_rpc_request(method, params)
+        return self._make_request(request)
+
+    def make_batch_request(
+        self, requests: List[Tuple[RPCEndpoint, Any]]
+    ) -> List[RPCResponse]:
+        self.logger.debug(f"Making batch request IPC. Path: {self.ipc_path}")
+        request_data = self.encode_batch_rpc_request(requests)
+        response = cast(List[RPCResponse], self._make_request(request_data))
+        return sort_batch_response_by_response_ids(response)
+
 
 # A valid JSON RPC response can only end in } or ] http://www.jsonrpc.org/specification
 def has_valid_json_rpc_ending(raw_response: bytes) -> bool:

web3/providers/legacy_websocket.py

@@ -10,9 +10,12 @@ from types import (
 )
 from typing import (
     Any,
+    List,
     Optional,
+    Tuple,
     Type,
     Union,
+    cast,
 )
 
 from eth_typing import (
@@ -25,6 +28,12 @@ from websockets.legacy.client import (
     WebSocketClientProtocol,
 )
 
+from web3._utils.batching import (
+    sort_batch_response_by_response_ids,
+)
+from web3._utils.caching import (
+    handle_request_caching,
+)
 from web3.exceptions import (
     Web3ValidationError,
 )
@@ -37,7 +46,7 @@ from web3.types import (
 )
 
 RESTRICTED_WEBSOCKET_KWARGS = {"uri", "loop"}
-DEFAULT_WEBSOCKET_TIMEOUT = 10
+DEFAULT_WEBSOCKET_TIMEOUT = 30
 
 
 def _start_event_loop(loop: asyncio.AbstractEventLoop) -> None:
@@ -91,7 +100,9 @@ class LegacyWebSocketProvider(JSONBaseProvider):
         endpoint_uri: Optional[Union[URI, str]] = None,
         websocket_kwargs: Optional[Any] = None,
         websocket_timeout: int = DEFAULT_WEBSOCKET_TIMEOUT,
+        **kwargs: Any,
     ) -> None:
+        super().__init__(**kwargs)
         self.endpoint_uri = URI(endpoint_uri)
         self.websocket_timeout = websocket_timeout
         if self.endpoint_uri is None:
@@ -110,7 +121,6 @@ class LegacyWebSocketProvider(JSONBaseProvider):
                     f"in websocket_kwargs, found: {found_restricted_keys}"
                 )
         self.conn = PersistentWebSocket(self.endpoint_uri, websocket_kwargs)
-        super().__init__()
 
     def __str__(self) -> str:
         return f"WS connection {self.endpoint_uri}"
@@ -124,6 +134,7 @@ class LegacyWebSocketProvider(JSONBaseProvider):
                 await asyncio.wait_for(conn.recv(), timeout=self.websocket_timeout)
             )
 
+    @handle_request_caching
     def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
         self.logger.debug(
             f"Making request WebSocket. URI: {self.endpoint_uri}, " f"Method: {method}"
@@ -133,3 +144,17 @@ class LegacyWebSocketProvider(JSONBaseProvider):
             self.coro_make_request(request_data), LegacyWebSocketProvider._loop
         )
         return future.result()
+
+    def make_batch_request(
+        self, requests: List[Tuple[RPCEndpoint, Any]]
+    ) -> List[RPCResponse]:
+        self.logger.debug(
+            f"Making batch request WebSocket. URI: {self.endpoint_uri}, "
+            f"Methods: {requests}"
+        )
+        request_data = self.encode_batch_rpc_request(requests)
+        future = asyncio.run_coroutine_threadsafe(
+            self.coro_make_request(request_data), LegacyWebSocketProvider._loop
+        )
+        response = cast(List[RPCResponse], future.result())
+        return sort_batch_response_by_response_ids(response)

web3/providers/persistent/__init__.py

@@ -13,3 +13,10 @@ from .async_ipc import (
 from .websocket import (
     WebSocketProvider,
 )
+
+__all__ = [
+    "PersistentConnectionProvider",
+    "PersistentConnection",
+    "AsyncIPCProvider",
+    "WebSocketProvider",
+]

web3/providers/persistent/async_ipc.py

@@ -1,9 +1,6 @@
 import asyncio
 import errno
 import json
-from json import (
-    JSONDecodeError,
-)
 import logging
 from pathlib import (
     Path,
@@ -16,10 +13,6 @@ from typing import (
     Union,
 )
 
-from eth_utils import (
-    to_text,
-)
-
 from web3.types import (
     RPCEndpoint,
     RPCResponse,
@@ -28,11 +21,11 @@ from web3.types import (
 from . import (
     PersistentConnectionProvider,
 )
-from ..._utils.caching import (
-    async_handle_request_caching,
-)
 from ...exceptions import (
+    PersistentConnectionClosedOK,
     ProviderConnectionError,
+    ReadBufferLimitReached,
+    Web3TypeError,
 )
 from ..ipc import (
     get_default_ipc_path,
@@ -40,7 +33,7 @@ from ..ipc import (
 
 
 async def async_get_ipc_socket(
-    ipc_path: str,
+    ipc_path: str, read_buffer_limit: int
 ) -> Tuple[asyncio.StreamReader, asyncio.StreamWriter]:
     if sys.platform == "win32":
         # On Windows named pipe is used. Simulate socket with it.
@@ -50,7 +43,7 @@ async def async_get_ipc_socket(
 
         return NamedPipe(ipc_path)
     else:
-        return await asyncio.open_unix_connection(ipc_path)
+        return await asyncio.open_unix_connection(ipc_path, limit=read_buffer_limit)
 
 
 class AsyncIPCProvider(PersistentConnectionProvider):
@@ -58,23 +51,24 @@ class AsyncIPCProvider(PersistentConnectionProvider):
 
     _reader: Optional[asyncio.StreamReader] = None
     _writer: Optional[asyncio.StreamWriter] = None
+    _decoder: json.JSONDecoder = json.JSONDecoder()
 
     def __init__(
         self,
         ipc_path: Optional[Union[str, Path]] = None,
-        max_connection_retries: int = 5,
+        read_buffer_limit: int = 20 * 1024 * 1024,  # 20 MB
         # `PersistentConnectionProvider` kwargs can be passed through
         **kwargs: Any,
     ) -> None:
+        # initialize the ipc_path before calling the super constructor
         if ipc_path is None:
             self.ipc_path = get_default_ipc_path()
         elif isinstance(ipc_path, str) or isinstance(ipc_path, Path):
             self.ipc_path = str(Path(ipc_path).expanduser().resolve())
         else:
-            raise TypeError("ipc_path must be of type string or pathlib.Path")
-
-        self._max_connection_retries = max_connection_retries
+            raise Web3TypeError("ipc_path must be of type string or pathlib.Path")
         super().__init__(**kwargs)
+        self.read_buffer_limit = read_buffer_limit
 
     def __str__(self) -> str:
         return f"<{self.__class__.__name__} {self.ipc_path}>"
@@ -84,81 +78,46 @@ class AsyncIPCProvider(PersistentConnectionProvider):
             return False
 
         try:
-            request_data = self.encode_rpc_request(
-                RPCEndpoint("web3_clientVersions"), []
-            )
-            self._writer.write(request_data)
-            current_request_id = json.loads(request_data)["id"]
-            await self._get_response_for_request_id(current_request_id, timeout=2)
+            await self.make_request(RPCEndpoint("web3_clientVersion"), [])
             return True
-        except (OSError, BrokenPipeError, ProviderConnectionError) as e:
+        except (OSError, ProviderConnectionError) as e:
             if show_traceback:
                 raise ProviderConnectionError(
                     f"Problem connecting to provider with error: {type(e)}: {e}"
                 )
             return False
 
-    async def connect(self) -> None:
-        _connection_attempts = 0
-        _backoff_rate_change = 1.75
-        _backoff_time = 1.75
-
-        while _connection_attempts != self._max_connection_retries:
-            try:
-                _connection_attempts += 1
-                self._reader, self._writer = await async_get_ipc_socket(self.ipc_path)
-                self._message_listener_task = asyncio.create_task(
-                    self._message_listener()
-                )
-                break
-            except OSError as e:
-                if _connection_attempts == self._max_connection_retries:
-                    raise ProviderConnectionError(
-                        f"Could not connect to endpoint: {self.endpoint_uri}. "
-                        f"Retries exceeded max of {self._max_connection_retries}."
-                    ) from e
-                self.logger.info(
-                    f"Could not connect to endpoint: {self.endpoint_uri}. Retrying in "
-                    f"{round(_backoff_time, 1)} seconds.",
-                    exc_info=True,
-                )
-                await asyncio.sleep(_backoff_time)
-                _backoff_time *= _backoff_rate_change
-
-    async def disconnect(self) -> None:
-        if self._writer and not self._writer.is_closing():
-            self._writer.close()
-            await self._writer.wait_closed()
-            self._writer = None
-            self.logger.debug(
-                f'Successfully disconnected from endpoint: "{self.endpoint_uri}'
-            )
-
-        try:
-            self._message_listener_task.cancel()
-            await self._message_listener_task
-            self._reader = None
-        except (asyncio.CancelledError, StopAsyncIteration):
-            pass
-
-        self._request_processor.clear_caches()
-
-    async def _reset_socket(self) -> None:
-        self._writer.close()
-        await self._writer.wait_closed()
-        self._reader, self._writer = await async_get_ipc_socket(self.ipc_path)
-
-    @async_handle_request_caching
-    async def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
-        request_data = self.encode_rpc_request(method, params)
-
+    async def socket_send(self, request_data: bytes) -> None:
         if self._writer is None:
             raise ProviderConnectionError(
                 "Connection to ipc socket has not been initiated for the provider."
             )
 
+        return await asyncio.wait_for(
+            self._socket_send(request_data), timeout=self.request_timeout
+        )
+
+    async def socket_recv(self) -> RPCResponse:
         try:
-            self._writer.write(request_data)
+            data = await self._reader.readline()
+        except ValueError as e:
+            if all(kw in str(e) for kw in ("limit", "chunk")):
+                raise ReadBufferLimitReached(
+                    f"Read buffer limit of `{self.read_buffer_limit}` bytes was "
+                    "reached. Consider increasing the ``read_buffer_limit`` on the "
+                    "AsyncIPCProvider."
+                ) from e
+            raise
+
+        if not data:
+            raise PersistentConnectionClosedOK("Socket reader received end of stream.")
+        return self.decode_rpc_response(data)
+
+    # -- private methods -- #
+
+    async def _socket_send(self, request_data: bytes) -> None:
+        try:
+            self._writer.write(request_data + b"\n")
             await self._writer.drain()
         except OSError as e:
             # Broken pipe
@@ -168,48 +127,29 @@ class AsyncIPCProvider(PersistentConnectionProvider):
                 self._writer.write(request_data)
                 await self._writer.drain()
 
-        current_request_id = json.loads(request_data)["id"]
-        response = await self._get_response_for_request_id(current_request_id)
-
-        return response
+    async def _reset_socket(self) -> None:
+        self._writer.close()
+        await self._writer.wait_closed()
+        self._reader, self._writer = await async_get_ipc_socket(
+            self.ipc_path, self.read_buffer_limit
+        )
 
-    async def _message_listener(self) -> None:
-        self.logger.info(
-            "IPC socket listener background task started. Storing all messages in "
-            "appropriate request processor queues / caches to be processed."
+    async def _provider_specific_connect(self) -> None:
+        self._reader, self._writer = await async_get_ipc_socket(
+            self.ipc_path, self.read_buffer_limit
         )
-        raw_message = ""
-        decoder = json.JSONDecoder()
-
-        while True:
-            # the use of sleep(0) seems to be the most efficient way to yield control
-            # back to the event loop to share the loop with other tasks.
-            await asyncio.sleep(0)
-
-            try:
-                raw_message += to_text(await self._reader.read(4096)).lstrip()
-
-                while raw_message:
-                    try:
-                        response, pos = decoder.raw_decode(raw_message)
-                    except JSONDecodeError:
-                        break
-
-                    is_subscription = response.get("method") == "eth_subscription"
-                    await self._request_processor.cache_raw_response(
-                        response, subscription=is_subscription
-                    )
-                    raw_message = raw_message[pos:].lstrip()
-            except Exception as e:
-                if not self.silence_listener_task_exceptions:
-                    loop = asyncio.get_event_loop()
-                    for task in asyncio.all_tasks(loop=loop):
-                        task.cancel()
-                    raise e
-
-                self.logger.error(
-                    "Exception caught in listener, error logging and keeping listener "
-                    f"background task alive.\n    error={e}"
-                )
-                # if only error logging, reset the ``raw_message`` buffer and continue
-                raw_message = ""
+
+    async def _provider_specific_disconnect(self) -> None:
+        # this should remain idempotent
+        if self._writer and not self._writer.is_closing():
+            self._writer.close()
+            await self._writer.wait_closed()
+            self._writer = None
+        if self._reader:
+            self._reader = None
+
+    async def _provider_specific_socket_reader(self) -> RPCResponse:
+        return await self.socket_recv()
+
+    def _error_log_listener_task_exception(self, e: Exception) -> None:
+        super()._error_log_listener_task_exception(e)