web3 7.0.0b5__py3-none-any.whl → 7.0.0b7__py3-none-any.whl

web3/providers/persistent/request_processor.py

@@ -2,20 +2,32 @@ import asyncio
 from copy import (
     copy,
 )
+import sys
 from typing import (
     TYPE_CHECKING,
     Any,
     Callable,
     Dict,
+    Generic,
     Optional,
     Tuple,
+    TypeVar,
+    Union,
 )
 
+from eth_utils.toolz import (
+    compose,
+)
+
+from web3._utils.batching import (
+    BATCH_REQUEST_ID,
+)
 from web3._utils.caching import (
     RequestInformation,
     generate_cache_key,
 )
 from web3.exceptions import (
+    TaskNotRunning,
     Web3ValueError,
 )
 from web3.types import (
@@ -31,6 +43,34 @@ if TYPE_CHECKING:
         PersistentConnectionProvider,
     )
 
+T = TypeVar("T")
+
+# TODO: This is an ugly hack for python 3.8. Remove this after we drop support for it
+#  and use `asyncio.Queue[T]` type directly in the `TaskReliantQueue` class.
+if sys.version_info >= (3, 9):
+
+    class _TaskReliantQueue(asyncio.Queue[T], Generic[T]):
+        pass
+
+else:
+
+    class _TaskReliantQueue(asyncio.Queue, Generic[T]):  # type: ignore
+        pass
+
+
+class TaskReliantQueue(_TaskReliantQueue[T]):
+    """
+    A queue that relies on a task to be running to process items in the queue.
+    """
+
+    async def get(self) -> T:
+        item = await super().get()
+        if isinstance(item, Exception):
+            # if the item is an exception, raise it so the task can handle this case
+            # more gracefully
+            raise item
+        return item
+
 
 class RequestProcessor:
     _subscription_queue_synced_with_ws_stream: bool = False
@@ -44,9 +84,9 @@ class RequestProcessor:
 
         self._request_information_cache: SimpleCache = SimpleCache(500)
         self._request_response_cache: SimpleCache = SimpleCache(500)
-        self._subscription_response_queue: asyncio.Queue[RPCResponse] = asyncio.Queue(
-            maxsize=subscription_response_queue_size
-        )
+        self._subscription_response_queue: TaskReliantQueue[
+            Union[RPCResponse, TaskNotRunning]
+        ] = TaskReliantQueue(maxsize=subscription_response_queue_size)
 
     @property
     def active_subscriptions(self) -> Dict[str, Any]:
@@ -62,7 +102,11 @@ class RequestProcessor:
         self,
         method: RPCEndpoint,
         params: Any,
-        response_formatters: Tuple[Callable[..., Any], ...],
+        response_formatters: Tuple[
+            Union[Dict[str, Callable[..., Any]], Callable[..., Any]],
+            Callable[..., Any],
+            Callable[..., Any],
+        ],
     ) -> Optional[str]:
         cached_requests_key = generate_cache_key((method, params))
         if cached_requests_key in self._provider._request_cache._data:
@@ -76,12 +120,17 @@ class RequestProcessor:
                 )
                 return None
 
-        # copy the request counter and find the next request id without incrementing
-        # since this is done when / if the request is successfully sent
-        request_id = next(copy(self._provider.request_counter))
-        cache_key = generate_cache_key(request_id)
+        if self._provider._is_batching:
+            # the _batch_request_counter is set when entering the context manager
+            current_request_id = self._provider._batch_request_counter
+            self._provider._batch_request_counter += 1
+        else:
+            # copy the request counter and find the next request id without incrementing
+            # since this is done when / if the request is successfully sent
+            current_request_id = next(copy(self._provider.request_counter))
+        cache_key = generate_cache_key(current_request_id)
 
-        self._bump_cache_if_key_present(cache_key, request_id)
+        self._bump_cache_if_key_present(cache_key, current_request_id)
 
         request_info = RequestInformation(
             method,
@@ -89,7 +138,7 @@ class RequestProcessor:
             response_formatters,
         )
         self._provider.logger.debug(
-            f"Caching request info:\n    request_id={request_id},\n"
+            f"Caching request info:\n    request_id={current_request_id},\n"
             f"    cache_key={cache_key},\n    request_info={request_info.__dict__}"
         )
         self._request_information_cache.cache(
@@ -153,9 +202,8 @@ class RequestProcessor:
                 # i.e. subscription request information remains in the cache
                 self._request_information_cache.get_cache_entry(cache_key)
             )
-
         else:
-            # retrieve the request info from the cache using the request id
+            # retrieve the request info from the cache using the response id
             cache_key = generate_cache_key(response["id"])
             if response in self._provider._request_cache._data.values():
                 request_info = (
@@ -184,6 +232,33 @@ class RequestProcessor:
 
         return request_info
 
+    def append_result_formatter_for_request(
+        self, request_id: int, result_formatter: Callable[..., Any]
+    ) -> None:
+        cache_key = generate_cache_key(request_id)
+        cached_request_info_for_id: RequestInformation = (
+            self._request_information_cache.get_cache_entry(cache_key)
+        )
+        if cached_request_info_for_id is not None:
+            (
+                current_result_formatters,
+                error_formatters,
+                null_result_formatters,
+            ) = cached_request_info_for_id.response_formatters
+            cached_request_info_for_id.response_formatters = (
+                compose(
+                    result_formatter,
+                    current_result_formatters,
+                ),
+                error_formatters,
+                null_result_formatters,
+            )
+        else:
+            self._provider.logger.debug(
+                f"No cached request info for response id `{request_id}`. Cannot "
+                f"append response formatter for response."
+            )
+
     def append_middleware_response_processor(
         self,
         response: RPCResponse,
@@ -218,7 +293,7 @@ class RequestProcessor:
     ) -> None:
         if subscription:
             if self._subscription_response_queue.full():
-                self._provider.logger.info(
+                self._provider.logger.debug(
                     "Subscription queue is full. Waiting for provider to consume "
                     "messages before caching."
                 )
@@ -229,6 +304,15 @@ class RequestProcessor:
                 f"Caching subscription response:\n    response={raw_response}"
             )
             await self._subscription_response_queue.put(raw_response)
+        elif isinstance(raw_response, list):
+            # Since only one batch should be in the cache at all times, we use a
+            # constant cache key for the batch response.
+            cache_key = generate_cache_key(BATCH_REQUEST_ID)
+            self._provider.logger.debug(
+                f"Caching batch response:\n    cache_key={cache_key},\n"
+                f"    response={raw_response}"
+            )
+            self._request_response_cache.cache(cache_key, raw_response)
         else:
             response_id = raw_response.get("id")
             cache_key = generate_cache_key(response_id)
@@ -289,6 +373,6 @@ class RequestProcessor:
         """Clear the request processor caches."""
         self._request_information_cache.clear()
         self._request_response_cache.clear()
-        self._subscription_response_queue = asyncio.Queue(
+        self._subscription_response_queue = TaskReliantQueue(
             maxsize=self._subscription_response_queue.maxsize
         )

web3/providers/persistent/websocket.py

@@ -5,8 +5,11 @@ import os
 from typing import (
     Any,
     Dict,
+    List,
     Optional,
+    Tuple,
     Union,
+    cast,
 )
 
 from eth_typing import (
@@ -25,6 +28,10 @@ from websockets.exceptions import (
     WebSocketException,
 )
 
+from web3._utils.batching import (
+    BATCH_REQUEST_ID,
+    sort_batch_response_by_response_ids,
+)
 from web3._utils.caching import (
     async_handle_request_caching,
 )
@@ -61,7 +68,6 @@ class WebSocketProvider(PersistentConnectionProvider):
     logger = logging.getLogger("web3.providers.WebSocketProvider")
     is_async: bool = True
 
-    _max_connection_retries: int = 5
     _ws: Optional[WebSocketClientProtocol] = None
 
     def __init__(
@@ -116,47 +122,13 @@ class WebSocketProvider(PersistentConnectionProvider):
                 ) from e
             return False
 
-    async def connect(self) -> None:
-        _connection_attempts = 0
-        _backoff_rate_change = 1.75
-        _backoff_time = 1.75
-
-        while _connection_attempts != self._max_connection_retries:
-            try:
-                _connection_attempts += 1
-                self._ws = await connect(self.endpoint_uri, **self.websocket_kwargs)
-                self._message_listener_task = asyncio.create_task(
-                    self._message_listener()
-                )
-                break
-            except WebSocketException as e:
-                if _connection_attempts == self._max_connection_retries:
-                    raise ProviderConnectionError(
-                        f"Could not connect to endpoint: {self.endpoint_uri}. "
-                        f"Retries exceeded max of {self._max_connection_retries}."
-                    ) from e
-                self.logger.info(
-                    f"Could not connect to endpoint: {self.endpoint_uri}. Retrying in "
-                    f"{round(_backoff_time, 1)} seconds.",
-                    exc_info=True,
-                )
-                await asyncio.sleep(_backoff_time)
-                _backoff_time *= _backoff_rate_change
+    async def _provider_specific_connect(self) -> None:
+        self._ws = await connect(self.endpoint_uri, **self.websocket_kwargs)
 
-    async def disconnect(self) -> None:
+    async def _provider_specific_disconnect(self) -> None:
         if self._ws is not None and not self._ws.closed:
             await self._ws.close()
             self._ws = None
-            self.logger.debug(
-                f'Successfully disconnected from endpoint: "{self.endpoint_uri}'
-            )
-
-        try:
-            self._message_listener_task.cancel()
-            await self._message_listener_task
-        except (asyncio.CancelledError, StopAsyncIteration):
-            pass
-        self._request_processor.clear_caches()
 
     @async_handle_request_caching
     async def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
@@ -176,34 +148,39 @@ class WebSocketProvider(PersistentConnectionProvider):
 
         return response
 
-    async def _message_listener(self) -> None:
-        self.logger.info(
-            "WebSocket listener background task started. Storing all messages in "
-            "appropriate request processor queues / caches to be processed."
+    async def make_batch_request(
+        self, requests: List[Tuple[RPCEndpoint, Any]]
+    ) -> List[RPCResponse]:
+        request_data = self.encode_batch_rpc_request(requests)
+
+        if self._ws is None:
+            raise ProviderConnectionError(
+                "Connection to websocket has not been initiated for the provider."
+            )
+
+        await asyncio.wait_for(
+            self._ws.send(request_data), timeout=self.request_timeout
+        )
+
+        response = cast(
+            List[RPCResponse],
+            await self._get_response_for_request_id(BATCH_REQUEST_ID),
         )
-        while True:
-            # the use of sleep(0) seems to be the most efficient way to yield control
-            # back to the event loop to share the loop with other tasks.
+        return response
+
+    async def _provider_specific_message_listener(self) -> None:
+        async for raw_message in self._ws:
             await asyncio.sleep(0)
 
-            try:
-                async for raw_message in self._ws:
-                    await asyncio.sleep(0)
-
-                    response = json.loads(raw_message)
-                    subscription = response.get("method") == "eth_subscription"
-                    await self._request_processor.cache_raw_response(
-                        response, subscription=subscription
-                    )
-            except Exception as e:
-                if not self.silence_listener_task_exceptions:
-                    loop = asyncio.get_event_loop()
-                    for task in asyncio.all_tasks(loop=loop):
-                        task.cancel()
-                    raise e
-
-                self.logger.error(
-                    "Exception caught in listener, error logging and keeping "
-                    "listener background task alive."
-                    f"\n    error={e.__class__.__name__}: {e}"
-                )
+            response = json.loads(raw_message)
+            if isinstance(response, list):
+                response = sort_batch_response_by_response_ids(response)
+
+            subscription = (
+                response.get("method") == "eth_subscription"
+                if not isinstance(response, list)
+                else False
+            )
+            await self._request_processor.cache_raw_response(
+                response, subscription=subscription
+            )

web3/providers/rpc/__init__.py

@@ -4,3 +4,8 @@ from .async_rpc import (
 from .rpc import (
     HTTPProvider,
 )
+
+__all__ = [
+    "AsyncHTTPProvider",
+    "HTTPProvider",
+]

web3/providers/rpc/async_rpc.py

@@ -4,9 +4,11 @@ from typing import (
     Any,
     Dict,
     Iterable,
+    List,
     Optional,
     Tuple,
     Union,
+    cast,
 )
 
 from aiohttp import (
@@ -27,19 +29,20 @@ from web3._utils.empty import (
 from web3._utils.http import (
     construct_user_agent,
 )
-from web3._utils.request import (
-    async_cache_and_return_session as _async_cache_and_return_session,
-    async_make_post_request,
-    get_default_http_endpoint,
-)
 from web3.types import (
     RPCEndpoint,
     RPCResponse,
 )
 
+from ..._utils.batching import (
+    sort_batch_response_by_response_ids,
+)
 from ..._utils.caching import (
     async_handle_request_caching,
 )
+from ..._utils.http_session_manager import (
+    HTTPSessionManager,
+)
 from ..async_base import (
     AsyncJSONBaseProvider,
 )
@@ -58,22 +61,29 @@ class AsyncHTTPProvider(AsyncJSONBaseProvider):
         self,
         endpoint_uri: Optional[Union[URI, str]] = None,
         request_kwargs: Optional[Any] = None,
-        exception_retry_configuration: Union[
-            ExceptionRetryConfiguration, Empty
+        exception_retry_configuration: Optional[
+            Union[ExceptionRetryConfiguration, Empty]
         ] = empty,
+        **kwargs: Any,
     ) -> None:
+        self._request_session_manager = HTTPSessionManager()
+
         if endpoint_uri is None:
-            self.endpoint_uri = get_default_http_endpoint()
+            self.endpoint_uri = (
+                self._request_session_manager.get_default_http_endpoint()
+            )
         else:
             self.endpoint_uri = URI(endpoint_uri)
 
         self._request_kwargs = request_kwargs or {}
         self._exception_retry_configuration = exception_retry_configuration
 
-        super().__init__()
+        super().__init__(**kwargs)
 
     async def cache_async_session(self, session: ClientSession) -> ClientSession:
-        return await _async_cache_and_return_session(self.endpoint_uri, session)
+        return await self._request_session_manager.async_cache_and_return_session(
+            self.endpoint_uri, session
+        )
 
     def __str__(self) -> str:
         return f"RPC connection {self.endpoint_uri}"
@@ -117,7 +127,7 @@ class AsyncHTTPProvider(AsyncJSONBaseProvider):
         ):
             for i in range(self.exception_retry_configuration.retries):
                 try:
-                    return await async_make_post_request(
+                    return await self._request_session_manager.async_make_post_request(
                         self.endpoint_uri, request_data, **self.get_request_kwargs()
                     )
                 except tuple(self.exception_retry_configuration.errors):
@@ -130,7 +140,7 @@ class AsyncHTTPProvider(AsyncJSONBaseProvider):
                         raise
             return None
         else:
-            return await async_make_post_request(
+            return await self._request_session_manager.async_make_post_request(
                 self.endpoint_uri, request_data, **self.get_request_kwargs()
             )
 
@@ -147,3 +157,15 @@ class AsyncHTTPProvider(AsyncJSONBaseProvider):
             f"Method: {method}, Response: {response}"
         )
         return response
+
+    async def make_batch_request(
+        self, batch_requests: List[Tuple[RPCEndpoint, Any]]
+    ) -> List[RPCResponse]:
+        self.logger.debug(f"Making batch request HTTP - uri: `{self.endpoint_uri}`")
+        request_data = self.encode_batch_rpc_request(batch_requests)
+        raw_response = await self._request_session_manager.async_make_post_request(
+            self.endpoint_uri, request_data, **self.get_request_kwargs()
+        )
+        self.logger.debug("Received batch response HTTP.")
+        responses_list = cast(List[RPCResponse], self.decode_rpc_response(raw_response))
+        return sort_batch_response_by_response_ids(responses_list)

web3/providers/rpc/rpc.py

@@ -5,9 +5,11 @@ from typing import (
     Any,
     Dict,
     Iterable,
+    List,
     Optional,
     Tuple,
     Union,
+    cast,
 )
 
 from eth_typing import (
@@ -25,19 +27,20 @@ from web3._utils.empty import (
 from web3._utils.http import (
     construct_user_agent,
 )
-from web3._utils.request import (
-    cache_and_return_session,
-    get_default_http_endpoint,
-    make_post_request,
-)
 from web3.types import (
     RPCEndpoint,
     RPCResponse,
 )
 
+from ..._utils.batching import (
+    sort_batch_response_by_response_ids,
+)
 from ..._utils.caching import (
     handle_request_caching,
 )
+from ..._utils.http_session_manager import (
+    HTTPSessionManager,
+)
 from ..base import (
     JSONBaseProvider,
 )
@@ -62,12 +65,17 @@ class HTTPProvider(JSONBaseProvider):
         endpoint_uri: Optional[Union[URI, str]] = None,
         request_kwargs: Optional[Any] = None,
         session: Optional[Any] = None,
-        exception_retry_configuration: Union[
-            ExceptionRetryConfiguration, Empty
+        exception_retry_configuration: Optional[
+            Union[ExceptionRetryConfiguration, Empty]
         ] = empty,
+        **kwargs: Any,
     ) -> None:
+        self._request_session_manager = HTTPSessionManager()
+
         if endpoint_uri is None:
-            self.endpoint_uri = get_default_http_endpoint()
+            self.endpoint_uri = (
+                self._request_session_manager.get_default_http_endpoint()
+            )
         else:
             self.endpoint_uri = URI(endpoint_uri)
 
@@ -75,9 +83,11 @@ class HTTPProvider(JSONBaseProvider):
         self._exception_retry_configuration = exception_retry_configuration
 
         if session:
-            cache_and_return_session(self.endpoint_uri, session)
+            self._request_session_manager.cache_and_return_session(
+                self.endpoint_uri, session
+            )
 
-        super().__init__()
+        super().__init__(**kwargs)
 
     def __str__(self) -> str:
         return f"RPC connection {self.endpoint_uri}"
@@ -125,7 +135,7 @@ class HTTPProvider(JSONBaseProvider):
         ):
             for i in range(self.exception_retry_configuration.retries):
                 try:
-                    return make_post_request(
+                    return self._request_session_manager.make_post_request(
                         self.endpoint_uri, request_data, **self.get_request_kwargs()
                     )
                 except tuple(self.exception_retry_configuration.errors) as e:
@@ -138,7 +148,7 @@ class HTTPProvider(JSONBaseProvider):
                         raise e
             return None
         else:
-            return make_post_request(
+            return self._request_session_manager.make_post_request(
                 self.endpoint_uri, request_data, **self.get_request_kwargs()
             )
 
@@ -155,3 +165,15 @@ class HTTPProvider(JSONBaseProvider):
             f"Method: {method}, Response: {response}"
         )
         return response
+
+    def make_batch_request(
+        self, batch_requests: List[Tuple[RPCEndpoint, Any]]
+    ) -> List[RPCResponse]:
+        self.logger.debug(f"Making batch request HTTP, uri: `{self.endpoint_uri}`")
+        request_data = self.encode_batch_rpc_request(batch_requests)
+        raw_response = self._request_session_manager.make_post_request(
+            self.endpoint_uri, request_data, **self.get_request_kwargs()
+        )
+        self.logger.debug("Received batch response HTTP.")
+        responses_list = cast(List[RPCResponse], self.decode_rpc_response(raw_response))
+        return sort_batch_response_by_response_ids(responses_list)

web3/providers/rpc/utils.py

@@ -19,9 +19,6 @@ REQUEST_RETRY_ALLOWLIST = [
     "evm",
     "eth_protocolVersion",
     "eth_syncing",
-    "eth_coinbase",
-    "eth_mining",
-    "eth_hashrate",
     "eth_chainId",
     "eth_gasPrice",
     "eth_accounts",

web3/tools/benchmark/main.py

@@ -108,13 +108,14 @@ def main(logger: logging.Logger, num_calls: int) -> None:
                 asyncio.set_event_loop(loop)
 
             # -- sync -- #
-            coinbase = w3_http.eth.coinbase
+            account = w3_http.eth.accounts[0]
 
             # -- async -- #
             async_w3_http = loop.run_until_complete(
                 build_async_w3_http(fixture.endpoint_uri)
             )
-            async_coinbase = loop.run_until_complete(async_w3_http.eth.coinbase)
+            async_accounts = loop.run_until_complete(async_w3_http.eth.accounts)
+            async_account = async_accounts[0]
 
             methods = [
                 {
@@ -126,17 +127,17 @@ def main(logger: logging.Logger, num_calls: int) -> None:
                 {
                     "name": "eth_sendTransaction",
                     "params": {},
-                    "exec": lambda w3_http=w3_http, coinbase=coinbase: w3_http.eth.send_transaction(  # noqa: E501
+                    "exec": lambda w3_http=w3_http, account=account: w3_http.eth.send_transaction(  # noqa: E501
                         {
                             "to": "0xd3CdA913deB6f67967B99D67aCDFa1712C293601",
-                            "from": coinbase,
+                            "from": account,
                             "value": Wei(1),
                         }
                     ),
-                    "async_exec": lambda async_w3_http=async_w3_http, async_coinbase=async_coinbase: async_w3_http.eth.send_transaction(  # noqa: E501
+                    "async_exec": lambda async_w3_http=async_w3_http, async_account=async_account: async_w3_http.eth.send_transaction(  # noqa: E501
                         {
                             "to": "0xd3CdA913deB6f67967B99D67aCDFa1712C293601",
-                            "from": async_coinbase,
+                            "from": async_account,
                             "value": Wei(1),
                         }
                     ),

web3/tools/benchmark/node.py

@@ -24,7 +24,7 @@ from web3.tools.benchmark.utils import (
     kill_proc_gracefully,
 )
 
-GETH_FIXTURE_ZIP = "geth-1.13.14-fixture.zip"
+GETH_FIXTURE_ZIP = "geth-1.14.5-fixture.zip"
 
 
 class GethBenchmarkFixture:

web3/types.py

@@ -9,6 +9,7 @@ from typing import (
     NewType,
     Optional,
     Sequence,
+    Tuple,
     Type,
     TypedDict,
     TypeVar,
@@ -43,8 +44,9 @@ if TYPE_CHECKING:
     )
 
 
-TReturn = TypeVar("TReturn")
+TFunc = TypeVar("TFunc", bound=Callable[..., Any])
 TParams = TypeVar("TParams")
+TReturn = TypeVar("TReturn")
 TValue = TypeVar("TValue")
 
 BlockParams = Literal["latest", "earliest", "pending", "safe", "finalized"]
@@ -318,7 +320,11 @@ class CreateAccessListResponse(TypedDict):
 
 
 MakeRequestFn = Callable[[RPCEndpoint, Any], RPCResponse]
+MakeBatchRequestFn = Callable[[List[Tuple[RPCEndpoint, Any]]], List[RPCResponse]]
 AsyncMakeRequestFn = Callable[[RPCEndpoint, Any], Coroutine[Any, Any, RPCResponse]]
+AsyncMakeBatchRequestFn = Callable[
+    [List[Tuple[RPCEndpoint, Any]]], Coroutine[Any, Any, List[RPCResponse]]
+]
 
 
 class FormattersDict(TypedDict, total=False):