web3 6.18.0__tar.gz → 6.20.0__tar.gz

{web3-6.18.0 → web3-6.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: web3
-Version: 6.18.0
+Version: 6.20.0
 Summary: web3.py
 Home-page: https://github.com/ethereum/web3.py
 Author: The Ethereum Foundation

{web3-6.18.0 → web3-6.20.0}/pyproject.toml

@@ -34,7 +34,7 @@ log_date_format = "%m-%d %H:%M:%S"
 [tool.towncrier]
 # Read https://github.com/ethereum/web3.py/blob/main/newsfragments/README.md for instructions
 package = "web3"
-filename = "docs/releases.rst"
+filename = "docs/release_notes.rst"
 directory = "newsfragments"
 underlines = ["-", "~", "^"]
 title_format = "web3.py v{version} ({project_date})"

{web3-6.18.0 → web3-6.20.0}/setup.py

@@ -51,7 +51,7 @@ with open("./README.md") as readme:
 setup(
     name="web3",
     # *IMPORTANT*: Don't manually change the version here. Use the 'bumpversion' utility.
-    version="6.18.0",
+    version="6.20.0",
     description="""web3.py""",
     long_description_content_type="text/markdown",
     long_description=long_description,

{web3-6.18.0 → web3-6.20.0}/web3/__init__.py

@@ -1,6 +1,15 @@
 from eth_account import Account  # noqa: E402
 import sys
 
+from web3.providers import (
+    AsyncBaseProvider,
+    AutoProvider,
+    BaseProvider,
+    JSONBaseProvider,
+    PersistentConnectionProvider,
+)
+
+
 if sys.version_info.major == 3 and sys.version_info.minor < 8:
     import pkg_resources
 
@@ -19,6 +28,7 @@ from web3.providers.async_rpc import (  # noqa: E402
     AsyncHTTPProvider,
 )
 from web3.providers.eth_tester import (  # noqa: E402
+    AsyncEthereumTesterProvider,
     EthereumTesterProvider,
 )
 from web3.providers.ipc import (  # noqa: E402
@@ -35,13 +45,21 @@ from web3.providers.websocket import (  # noqa: E402
 
 __all__ = [
     "__version__",
+    "Account",
+    # web3:
     "AsyncWeb3",
     "Web3",
+    # providers:
+    "AsyncBaseProvider",
+    "AsyncEthereumTesterProvider",
+    "AsyncHTTPProvider",
+    "AutoProvider",
+    "BaseProvider",
+    "EthereumTesterProvider",
     "HTTPProvider",
     "IPCProvider",
+    "JSONBaseProvider",
+    "PersistentConnectionProvider",
     "WebsocketProvider",
     "WebsocketProviderV2",
-    "EthereumTesterProvider",
-    "Account",
-    "AsyncHTTPProvider",
 ]

{web3-6.18.0 → web3-6.20.0}/web3/_utils/module_testing/module_testing_utils.py

@@ -193,6 +193,12 @@ class WebsocketMessageStreamMock:
         self.messages = deque(messages) if messages else deque()
         self.raise_exception = raise_exception
 
+    def __await__(self) -> Generator[Any, Any, "Self"]:
+        async def __async_init__() -> "Self":
+            return self
+
+        return __async_init__().__await__()
+
     def __aiter__(self) -> "Self":
         return self
 
@@ -205,6 +211,13 @@ class WebsocketMessageStreamMock:
 
         return self.messages.popleft()
 
+    @staticmethod
+    async def pong() -> Literal[False]:
+        return False
+
+    async def connect(self) -> None:
+        pass
+
     async def send(self, data: bytes) -> None:
         pass
 

{web3-6.18.0 → web3-6.20.0}/web3/exceptions.py

@@ -1,6 +1,7 @@
 import datetime
 import time
 from typing import (
+    TYPE_CHECKING,
     Any,
     Dict,
     Optional,
@@ -15,6 +16,9 @@ from web3.types import (
     BlockData,
 )
 
+if TYPE_CHECKING:
+    import asyncio
+
 
 class Web3Exception(Exception):
     """
@@ -341,7 +345,21 @@ class BadResponseFormat(Web3Exception):
     Raised when a JSON-RPC response comes back in an unexpected format
     """
 
-    pass
+
+class TaskNotRunning(Web3Exception):
+    """
+    Used to signal between asyncio contexts that a task that is being awaited
+    is not currently running.
+    """
+
+    def __init__(
+        self, task: "asyncio.Task[Any]", message: Optional[str] = None
+    ) -> None:
+        self.task = task
+        if message is None:
+            message = f"Task {task} is not running."
+        self.message = message
+        super().__init__(message)
 
 
 class MethodUnavailable(Web3Exception):

{web3-6.18.0 → web3-6.20.0}/web3/main.py

@@ -573,12 +573,9 @@ class _PersistentConnectionWeb3(AsyncWeb3):
 
     # async for w3 in w3.persistent_websocket(provider)
     async def __aiter__(self) -> AsyncIterator[Self]:
-        if not await self.provider.is_connected():
-            await self.provider.connect()
-
+        provider = self.provider
         while True:
-            try:
-                yield self
-            except Exception:
-                # provider should handle connection / reconnection
-                continue
+            await provider.connect()
+            yield self
+            provider.logger.error("Connection interrupted, attempting to reconnect...")
+            await provider.disconnect()

{web3-6.18.0 → web3-6.20.0}/web3/manager.py

@@ -35,6 +35,7 @@ from web3.exceptions import (
     BadResponseFormat,
     MethodUnavailable,
     ProviderConnectionError,
+    TaskNotRunning,
 )
 from web3.middleware import (
     abi_middleware,
@@ -377,14 +378,24 @@ class RequestManager:
             raise ProviderConnectionError("No listener found for websocket connection.")
 
         while True:
-            response = await self._request_processor.pop_raw_response(subscription=True)
-            if (
-                response is not None
-                and response.get("params", {}).get("subscription")
-                in self._request_processor.active_subscriptions
-            ):
-                # if response is an active subscription response, process it
-                yield await self._process_ws_response(response)
+            try:
+                response = await self._request_processor.pop_raw_response(
+                    subscription=True
+                )
+                if (
+                    response is not None
+                    and response.get("params", {}).get("subscription")
+                    in self._request_processor.active_subscriptions
+                ):
+                    # if response is an active subscription response, process it
+                    yield await self._process_ws_response(response)
+            except TaskNotRunning:
+                self._provider._handle_listener_task_exceptions()
+                self.logger.error(
+                    "Message listener background task has stopped unexpectedly. "
+                    "Stopping message stream."
+                )
+                raise StopAsyncIteration
 
     async def _process_ws_response(self, response: RPCResponse) -> RPCResponse:
         provider = cast(PersistentConnectionProvider, self._provider)

{web3-6.18.0 → web3-6.20.0}/web3/providers/__init__.py

@@ -8,6 +8,10 @@ from .base import (
     BaseProvider,
     JSONBaseProvider,
 )
+from .eth_tester import (
+    AsyncEthereumTesterProvider,
+    EthereumTesterProvider,
+)
 from .ipc import (
     IPCProvider,
 )
@@ -24,3 +28,18 @@ from .persistent import (
 from .auto import (
     AutoProvider,
 )
+
+__all__ = [
+    "AsyncBaseProvider",
+    "AsyncEthereumTesterProvider",
+    "AsyncHTTPProvider",
+    "AutoProvider",
+    "BaseProvider",
+    "EthereumTesterProvider",
+    "HTTPProvider",
+    "IPCProvider",
+    "JSONBaseProvider",
+    "PersistentConnectionProvider",
+    "WebsocketProvider",
+    "WebsocketProviderV2",
+]

web3-6.20.0/web3/providers/persistent.py

@@ -0,0 +1,214 @@
+from abc import (
+    ABC,
+)
+import asyncio
+import logging
+from typing import (
+    Optional,
+)
+
+from websockets import (
+    ConnectionClosed,
+    WebSocketClientProtocol,
+    WebSocketException,
+)
+
+from web3._utils.caching import (
+    generate_cache_key,
+)
+from web3.exceptions import (
+    ProviderConnectionError,
+    TaskNotRunning,
+    TimeExhausted,
+)
+from web3.providers.async_base import (
+    AsyncJSONBaseProvider,
+)
+from web3.providers.websocket.request_processor import (
+    RequestProcessor,
+)
+from web3.types import (
+    RPCId,
+    RPCResponse,
+)
+
+DEFAULT_PERSISTENT_CONNECTION_TIMEOUT = 50.0
+
+
+class PersistentConnectionProvider(AsyncJSONBaseProvider, ABC):
+    logger = logging.getLogger("web3.providers.PersistentConnectionProvider")
+    has_persistent_connection = True
+    endpoint_uri: Optional[str] = None
+
+    _max_connection_retries: int = 5
+
+    _ws: Optional[WebSocketClientProtocol] = None
+    _request_processor: RequestProcessor
+    _message_listener_task: Optional["asyncio.Task[None]"] = None
+    _listen_event: asyncio.Event = asyncio.Event()
+
+    def __init__(
+        self,
+        request_timeout: float = DEFAULT_PERSISTENT_CONNECTION_TIMEOUT,
+        subscription_response_queue_size: int = 500,
+        request_information_cache_size: int = 500,
+        silence_listener_task_exceptions: bool = False,
+    ) -> None:
+        super().__init__()
+        self._request_processor = RequestProcessor(
+            self,
+            subscription_response_queue_size=subscription_response_queue_size,
+            request_information_cache_size=request_information_cache_size,
+        )
+        self.request_timeout = request_timeout
+        self.silence_listener_task_exceptions = silence_listener_task_exceptions
+
+    async def connect(self) -> None:
+        _connection_attempts = 0
+        _backoff_rate_change = 1.75
+        _backoff_time = 1.75
+
+        while _connection_attempts != self._max_connection_retries:
+            try:
+                _connection_attempts += 1
+                self.logger.info(f"Connecting to: {self.endpoint_uri}")
+                await self._provider_specific_connect()
+                self._message_listener_task = asyncio.create_task(
+                    self._message_listener()
+                )
+                self._message_listener_task.add_done_callback(
+                    self._message_listener_callback
+                )
+                self.logger.info(f"Successfully connected to: {self.endpoint_uri}")
+                break
+            except (WebSocketException, OSError) as e:
+                if _connection_attempts == self._max_connection_retries:
+                    raise ProviderConnectionError(
+                        f"Could not connect to: {self.endpoint_uri}. "
+                        f"Retries exceeded max of {self._max_connection_retries}."
+                    ) from e
+                self.logger.info(
+                    f"Could not connect to: {self.endpoint_uri}. "
+                    f"Retrying in {round(_backoff_time, 1)} seconds.",
+                    exc_info=True,
+                )
+                await asyncio.sleep(_backoff_time)
+                _backoff_time *= _backoff_rate_change
+
+    async def disconnect(self) -> None:
+        try:
+            if self._message_listener_task:
+                self._message_listener_task.cancel()
+                await self._message_listener_task
+        except (asyncio.CancelledError, StopAsyncIteration, ConnectionClosed):
+            pass
+        finally:
+            self._message_listener_task = None
+            self.logger.info("Message listener background task successfully shut down.")
+
+        await self._provider_specific_disconnect()
+        self._request_processor.clear_caches()
+        self.logger.info(f"Successfully disconnected from: {self.endpoint_uri}")
+
+    # -- private methods -- #
+
+    async def _provider_specific_connect(self) -> None:
+        raise NotImplementedError("Must be implemented by subclasses")
+
+    async def _provider_specific_disconnect(self) -> None:
+        raise NotImplementedError("Must be implemented by subclasses")
+
+    async def _provider_specific_message_listener(self) -> None:
+        raise NotImplementedError("Must be implemented by subclasses")
+
+    def _message_listener_callback(
+        self, message_listener_task: "asyncio.Task[None]"
+    ) -> None:
+        # Puts a `TaskNotRunning` in the queue to signal the end of the listener task
+        # to any running subscription streams that are awaiting a response.
+        self._request_processor._subscription_response_queue.put_nowait(
+            TaskNotRunning(message_listener_task)
+        )
+
+    async def _message_listener(self) -> None:
+        self.logger.info(
+            f"{self.__class__.__qualname__} listener background task started. Storing "
+            "all messages in appropriate request processor queues / caches to be "
+            "processed."
+        )
+        while True:
+            # the use of sleep(0) seems to be the most efficient way to yield control
+            # back to the event loop to share the loop with other tasks.
+            await asyncio.sleep(0)
+            try:
+                await self._provider_specific_message_listener()
+            except Exception as e:
+                if not self.silence_listener_task_exceptions:
+                    raise e
+                else:
+                    self._error_log_listener_task_exception(e)
+
+    def _error_log_listener_task_exception(self, e: Exception) -> None:
+        """
+        When silencing listener task exceptions, this method is used to log the
+        exception and keep the listener task alive. Override this method to fine-tune
+        error logging behavior for the implementation class.
+        """
+        self.logger.error(
+            "Exception caught in listener, error logging and keeping "
+            "listener background task alive."
+            f"\n    error={e.__class__.__name__}: {e}"
+        )
+
+    def _handle_listener_task_exceptions(self) -> None:
+        """
+        Should be called every time a `PersistentConnectionProvider` is polling for
+        messages in the main loop. If the message listener task has completed and an
+        exception was recorded, raise the exception in the main loop.
+        """
+        msg_listener_task = getattr(self, "_message_listener_task", None)
+        if (
+            msg_listener_task
+            and msg_listener_task.done()
+            and msg_listener_task.exception()
+        ):
+            raise msg_listener_task.exception()
+
+    async def _get_response_for_request_id(
+        self, request_id: RPCId, timeout: Optional[float] = None
+    ) -> RPCResponse:
+        if timeout is None:
+            timeout = self.request_timeout
+
+        async def _match_response_id_to_request_id() -> RPCResponse:
+            request_cache_key = generate_cache_key(request_id)
+
+            while True:
+                # check if an exception was recorded in the listener task and raise it
+                # in the main loop if so
+                self._handle_listener_task_exceptions()
+
+                if request_cache_key in self._request_processor._request_response_cache:
+                    self.logger.debug(
+                        f"Popping response for id {request_id} from cache."
+                    )
+                    popped_response = await self._request_processor.pop_raw_response(
+                        cache_key=request_cache_key,
+                    )
+                    return popped_response
+                else:
+                    await asyncio.sleep(0)
+
+        try:
+            # Add the request timeout around the while loop that checks the request
+            # cache. If the request is not in the cache within the request_timeout,
+            # raise ``TimeExhausted``.
+            return await asyncio.wait_for(_match_response_id_to_request_id(), timeout)
+        except asyncio.TimeoutError:
+            raise TimeExhausted(
+                f"Timed out waiting for response with request id `{request_id}` after "
+                f"{self.request_timeout} second(s). This may be due to the provider "
+                "not returning a response with the same id that was sent in the "
+                "request or an exception raised during the request was caught and "
+                "allowed to continue."
+            )

{web3-6.18.0 → web3-6.20.0}/web3/providers/websocket/request_processor.py

@@ -2,19 +2,26 @@ import asyncio
 from copy import (
     copy,
 )
+import sys
 from typing import (
     TYPE_CHECKING,
     Any,
     Callable,
     Dict,
+    Generic,
     Optional,
     Tuple,
+    TypeVar,
+    Union,
 )
 
 from web3._utils.caching import (
     RequestInformation,
     generate_cache_key,
 )
+from web3.exceptions import (
+    TaskNotRunning,
+)
 from web3.types import (
     RPCEndpoint,
     RPCResponse,
@@ -28,6 +35,34 @@ if TYPE_CHECKING:
         PersistentConnectionProvider,
     )
 
+T = TypeVar("T")
+
+# TODO: This is an ugly hack for python 3.8. Remove this after we drop support for it
+#  and use `asyncio.Queue[T]` type directly in the `TaskReliantQueue` class.
+if sys.version_info >= (3, 9):
+
+    class _TaskReliantQueue(asyncio.Queue[T], Generic[T]):
+        pass
+
+else:
+
+    class _TaskReliantQueue(asyncio.Queue, Generic[T]):  # type: ignore
+        pass
+
+
+class TaskReliantQueue(_TaskReliantQueue[T]):
+    """
+    A queue that relies on a task to be running to process items in the queue.
+    """
+
+    async def get(self) -> T:
+        item = await super().get()
+        if isinstance(item, Exception):
+            # if the item is an exception, raise it so the task can handle this case
+            # more gracefully
+            raise item
+        return item
+
 
 class RequestProcessor:
     _subscription_queue_synced_with_ws_stream: bool = False
@@ -41,9 +76,9 @@ class RequestProcessor:
         self._provider = provider
 
         self._request_response_cache: SimpleCache = SimpleCache(500)
-        self._subscription_response_queue: asyncio.Queue[RPCResponse] = asyncio.Queue(
-            maxsize=subscription_response_queue_size
-        )
+        self._subscription_response_queue: TaskReliantQueue[
+            Union[RPCResponse, TaskNotRunning]
+        ] = TaskReliantQueue(maxsize=subscription_response_queue_size)
         self._request_information_cache: SimpleCache = SimpleCache(
             request_information_cache_size
         )
@@ -203,7 +238,7 @@ class RequestProcessor:
     ) -> None:
         if subscription:
             if self._subscription_response_queue.full():
-                self._provider.logger.info(
+                self._provider.logger.debug(
                     "Subscription queue is full. Waiting for provider to consume "
                     "messages before caching."
                 )
@@ -276,6 +311,6 @@ class RequestProcessor:
 
         self._request_information_cache.clear()
         self._request_response_cache.clear()
-        self._subscription_response_queue = asyncio.Queue(
+        self._subscription_response_queue = TaskReliantQueue(
             maxsize=self._subscription_response_queue.maxsize
         )

web3-6.20.0/web3/providers/websocket/websocket_v2.py

@@ -0,0 +1,146 @@
+import asyncio
+import json
+import logging
+import os
+from typing import (
+    Any,
+    Dict,
+    Optional,
+    Union,
+)
+
+from eth_typing import (
+    URI,
+)
+from toolz import (
+    merge,
+)
+from websockets.client import (
+    connect,
+)
+from websockets.exceptions import (
+    WebSocketException,
+)
+
+from web3.exceptions import (
+    ProviderConnectionError,
+    Web3ValidationError,
+)
+from web3.providers.persistent import (
+    PersistentConnectionProvider,
+)
+from web3.types import (
+    RPCEndpoint,
+    RPCResponse,
+)
+
+DEFAULT_PING_INTERVAL = 30  # 30 seconds
+DEFAULT_PING_TIMEOUT = 300  # 5 minutes
+
+VALID_WEBSOCKET_URI_PREFIXES = {"ws://", "wss://"}
+RESTRICTED_WEBSOCKET_KWARGS = {"uri", "loop"}
+DEFAULT_WEBSOCKET_KWARGS = {
+    # set how long to wait between pings from the server
+    "ping_interval": DEFAULT_PING_INTERVAL,
+    # set how long to wait without a pong response before closing the connection
+    "ping_timeout": DEFAULT_PING_TIMEOUT,
+}
+
+
+def get_default_endpoint() -> URI:
+    return URI(os.environ.get("WEB3_WS_PROVIDER_URI", "ws://127.0.0.1:8546"))
+
+
+class WebsocketProviderV2(PersistentConnectionProvider):
+    logger = logging.getLogger("web3.providers.WebsocketProviderV2")
+    is_async: bool = True
+
+    def __init__(
+        self,
+        endpoint_uri: Optional[Union[URI, str]] = None,
+        websocket_kwargs: Optional[Dict[str, Any]] = None,
+        silence_listener_task_exceptions: bool = False,
+        # `PersistentConnectionProvider` kwargs can be passed through
+        **kwargs: Any,
+    ) -> None:
+        self.endpoint_uri = (
+            URI(endpoint_uri) if endpoint_uri is not None else get_default_endpoint()
+        )
+
+        if not any(
+            self.endpoint_uri.startswith(prefix)
+            for prefix in VALID_WEBSOCKET_URI_PREFIXES
+        ):
+            raise Web3ValidationError(
+                "WebSocket endpoint uri must begin with 'ws://' or 'wss://': "
+                f"{self.endpoint_uri}"
+            )
+
+        if websocket_kwargs is not None:
+            found_restricted_keys = set(websocket_kwargs).intersection(
+                RESTRICTED_WEBSOCKET_KWARGS
+            )
+            if found_restricted_keys:
+                raise Web3ValidationError(
+                    "Found restricted keys for websocket_kwargs: "
+                    f"{found_restricted_keys}."
+                )
+
+        self.websocket_kwargs = merge(DEFAULT_WEBSOCKET_KWARGS, websocket_kwargs or {})
+
+        super().__init__(
+            silence_listener_task_exceptions=silence_listener_task_exceptions, **kwargs
+        )
+
+    def __str__(self) -> str:
+        return f"WebSocket connection: {self.endpoint_uri}"
+
+    async def is_connected(self, show_traceback: bool = False) -> bool:
+        if not self._ws:
+            return False
+
+        try:
+            await self._ws.pong()
+            return True
+
+        except WebSocketException as e:
+            if show_traceback:
+                raise ProviderConnectionError(
+                    f"Error connecting to endpoint: '{self.endpoint_uri}'"
+                ) from e
+            return False
+
+    async def _provider_specific_connect(self) -> None:
+        self._ws = await connect(self.endpoint_uri, **self.websocket_kwargs)
+
+    async def _provider_specific_disconnect(self) -> None:
+        if self._ws is not None and not self._ws.closed:
+            await self._ws.close()
+            self._ws = None
+
+    async def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
+        request_data = self.encode_rpc_request(method, params)
+
+        if self._ws is None:
+            raise ProviderConnectionError(
+                "Connection to websocket has not been initiated for the provider."
+            )
+
+        await asyncio.wait_for(
+            self._ws.send(request_data), timeout=self.request_timeout
+        )
+
+        current_request_id = json.loads(request_data)["id"]
+        response = await self._get_response_for_request_id(current_request_id)
+
+        return response
+
+    async def _provider_specific_message_listener(self) -> None:
+        async for raw_message in self._ws:
+            await asyncio.sleep(0)
+
+            response = json.loads(raw_message)
+            subscription = response.get("method") == "eth_subscription"
+            await self._request_processor.cache_raw_response(
+                response, subscription=subscription
+            )