ipor-fusion 4.0.0.dev114__tar.gz → 4.0.0.dev118__tar.gz

{ipor_fusion-4.0.0.dev114 → ipor_fusion-4.0.0.dev118}/AGENTS.md

@@ -394,7 +394,9 @@ byte, e.g. Ebisu ZAPPER/REGISTRY).
 role-accounts, oracle-mapping). `vault info` runs independent fetch pipelines
 (vault/asset, fees, WithdrawManager, markets, roles) concurrently on a
 `ThreadPoolExecutor`, each a short chain of `Multicall3` batches; keep new
-reads inside a batch rather than adding per-call round trips. Contract names,
+reads inside a batch rather than adding per-call round trips. A `from_url` context
+bounds requests in flight (`max_concurrent_requests`, default 8) whatever the pool
+nesting, so a new fan-out does not need its own limit. Contract names,
 token symbols and decimals and deployment blocks are cached per chain in
 `$XDG_CACHE_HOME/ipor-fusion/contract_cache.json` (`<chain>:<address>:<field>`
 keys); tests redirect it with `monkeypatch.setenv("XDG_CACHE_HOME", ...)` or

{ipor_fusion-4.0.0.dev114 → ipor_fusion-4.0.0.dev118}/MIGRATION_4.md

@@ -98,6 +98,12 @@ converted amounts `Amount`. A custom `VaultState` annotates the same.
 now shown in share decimals labelled `shares` (``fusion vault info``) and its JSON
 ``formatted`` uses `share_decimals`; it was formatted with the asset's decimals.
 
+`Web3Context.from_url(..., max_concurrent_requests=8)` bounds the RPC requests in
+flight across every thread using the context (``None`` lifts the bound); the vault
+reads fan out on nested thread pools, and the bound is what a rate-limited provider
+sees. A context built from your own `Web3` object is not bounded unless you add the
+same middleware.
+
 `ensure_supported_chain` takes `ChainId`. `fetch_unpriceable_priced_tokens` takes the oracle and
 underlying as `ChecksumAddress`. `resolve_token_symbol`, `resolve_token_decimals`,
 `get_contract_name` and `get_deployment_tx` take `HexAddress` (a checksum address

{ipor_fusion-4.0.0.dev114 → ipor_fusion-4.0.0.dev118}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ipor-fusion
-Version: 4.0.0.dev114
+Version: 4.0.0.dev118
 Summary: The IPOR Fusion SDK for Python
 Project-URL: homepage, https://ipor.io
 Project-URL: repository, https://github.com/IPOR-Labs/ipor-fusion.py

{ipor_fusion-4.0.0.dev114 → ipor_fusion-4.0.0.dev118}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ipor-fusion"
-version = "4.0.0.dev114"
+version = "4.0.0.dev118"
 description = "The IPOR Fusion SDK for Python"
 authors = [
     { name = "Adam Mydlarz", email = "amydlarz@gmail.com" },

{ipor_fusion-4.0.0.dev114 → ipor_fusion-4.0.0.dev118}/src/ipor_fusion/core/context.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 import copy
 import logging
+import threading
+from typing import Any
 
 from eth_account import Account
 from eth_typing import ChecksumAddress, HexStr
@@ -11,12 +13,22 @@ from requests import ConnectionError as RequestsConnectionError
 from requests import HTTPError, Timeout
 from web3 import Web3
 from web3.exceptions import ContractLogicError, Web3RPCError
-from web3.middleware import ExtraDataToPOAMiddleware
+from web3.middleware import ExtraDataToPOAMiddleware, Web3Middleware
 from web3.providers.rpc.utils import (
     REQUEST_RETRY_ALLOWLIST,
     ExceptionRetryConfiguration,
 )
-from web3.types import BlockIdentifier, LogReceipt, TxParams, TxReceipt, Wei
+from web3.types import (
+    BlockIdentifier,
+    LogReceipt,
+    MakeBatchRequestFn,
+    MakeRequestFn,
+    RPCEndpoint,
+    RPCResponse,
+    TxParams,
+    TxReceipt,
+    Wei,
+)
 
 from ipor_fusion.cache import ContractCache, default_contract_cache
 from ipor_fusion.chains import GET_LOGS_RANGE_HINTS
@@ -70,6 +82,11 @@ class Web3Context:
     # multi-call read (vault fetch, health check) fans that out into minutes of
     # wall clock. Long-running services should pass something tighter.
     DEFAULT_RPC_TIMEOUT_S = 30.0
+    # RPC requests in flight at once through a `from_url` context. The vault
+    # reads fan out on nested thread pools; this bounds what reaches the
+    # provider, whatever the nesting, so a rate-limited endpoint sees a steady
+    # stream instead of bursts.
+    DEFAULT_MAX_CONCURRENT_REQUESTS = 8
 
     def __init__(
         self,
@@ -142,7 +159,7 @@ class Web3Context:
         return self._signer
 
     @classmethod
-    def from_url(
+    def from_url(  # proctor: ignore[primitives-too-many-params]
         cls,
         url: str,
         *,
@@ -153,7 +170,11 @@ class Web3Context:
         log_scan_max_requests: int | None = DEFAULT_LOG_SCAN_MAX_REQUESTS,
         log_source: LogSource | None = None,
         cache: ContractCache | None = None,
+        max_concurrent_requests: int | None = DEFAULT_MAX_CONCURRENT_REQUESTS,
     ) -> Web3Context:
+        """A context on an HTTP provider at ``url``. ``max_concurrent_requests``
+        bounds the RPC requests in flight across every thread using it
+        (``None``: unbounded)."""
         web3 = Web3(
             Web3.HTTPProvider(
                 url,
@@ -171,6 +192,15 @@ class Web3Context:
         # with a ValueError. The middleware moves it to proofOfAuthorityData
         # and leaves every other block untouched.
         web3.middleware_onion.inject(ExtraDataToPOAMiddleware, layer=0)
+        if max_concurrent_requests is not None:
+            # Innermost (layer 0, injected last): a slot covers only the provider
+            # call, so a middleware that issues its own request (the gas price
+            # strategy reads a block) never waits on the slot it already holds.
+            web3.middleware_onion.inject(
+                _in_flight_limit(max_concurrent_requests),
+                name="in_flight_limit",
+                layer=0,
+            )
         chain_id = ChainId(web3.eth.chain_id)
 
         return cls(
@@ -385,3 +415,33 @@ class Web3Context:
     @staticmethod
     def _percent_of(value: int, percentage: int) -> int:
         return value * percentage // 100
+
+
+def _in_flight_limit(limit: int) -> type[Web3Middleware]:
+    """A middleware class (web3 instantiates it per Web3) holding a semaphore
+    slot for the duration of each provider call; a JSON-RPC batch is one HTTP
+    request, so it takes one slot whatever its item count."""
+    if limit < 1:
+        raise ValueError(f"max_concurrent_requests must be at least 1, got {limit}")
+    slots = threading.BoundedSemaphore(limit)
+
+    class InFlightLimit(Web3Middleware):
+        def wrap_make_request(self, make_request: MakeRequestFn) -> MakeRequestFn:
+            def limited(method: RPCEndpoint, params: Any) -> RPCResponse:
+                with slots:
+                    return make_request(method, params)
+
+            return limited
+
+        def wrap_make_batch_request(
+            self, make_batch_request: MakeBatchRequestFn
+        ) -> MakeBatchRequestFn:
+            def limited(
+                requests_info: list[tuple[RPCEndpoint, Any]],
+            ) -> list[RPCResponse] | RPCResponse:
+                with slots:
+                    return make_batch_request(requests_info)
+
+            return limited
+
+    return InFlightLimit

{ipor_fusion-4.0.0.dev114 → ipor_fusion-4.0.0.dev118}/src/ipor_fusion/vault_erc20.py

@@ -5,21 +5,21 @@ middleware price per token, valued in the underlying asset. Part of the
 from __future__ import annotations
 
 from collections.abc import Callable, Mapping, Sequence
-from concurrent.futures import Future, ThreadPoolExecutor
 from dataclasses import dataclass
-from functools import partial
+from typing import Any
 
 from eth_typing import ChecksumAddress, HexAddress
 from web3 import Web3
 
 from ipor_fusion.core.context import Web3Context
+from ipor_fusion.core.contract import Call
 from ipor_fusion.core.erc20 import ERC20
+from ipor_fusion.core.multicall import Multicall3
 from ipor_fusion.core.oracle import PriceOracleMiddleware
 from ipor_fusion.core.plasma_vault import BalanceFuse
 from ipor_fusion.market_ids import IporFusionMarkets
 from ipor_fusion.substrates import decode_substrate
 from ipor_fusion.types import MarketId, PlasmaVaultAddress
-from ipor_fusion.vault_reads import _safe_call
 
 
 @dataclass(frozen=True)
@@ -113,36 +113,32 @@ def _read_tokens(
     addresses: list[HexAddress],
     resolve_symbol: Callable[[HexAddress], str],
 ) -> list[_TokenRead]:
-    with ThreadPoolExecutor() as pool:
-        futures: list[tuple[HexAddress, Future, Future, Future, Future]] = []
-        for address in addresses:
-            checksum = Web3.to_checksum_address(address)
-            token = ERC20(checksum)
-            futures.append(
-                (
-                    address,
-                    pool.submit(resolve_symbol, address),
-                    pool.submit(_safe_call, partial(token.decimals().call, ctx)),
-                    pool.submit(_safe_call, partial(token.balance_of(vault).call, ctx)),
-                    pool.submit(
-                        _safe_call,
-                        partial(oracle.get_asset_price(checksum).call, ctx),
-                    ),
-                )
+    """Every token's decimals, the vault's balance and the middleware price in
+    one Multicall3 batch (a read that fails is ``None``); symbols through
+    ``resolve_symbol``."""
+    calls: list[Call[Any]] = []
+    for address in addresses:
+        checksum = Web3.to_checksum_address(address)
+        token = ERC20(checksum)
+        calls += [
+            token.decimals(),
+            token.balance_of(vault),
+            oracle.get_asset_price(checksum),
+        ]
+    results = Multicall3(ctx).try_aggregate(calls)
+    reads: list[_TokenRead] = []
+    for index, address in enumerate(addresses):
+        decimals, balance, price = results[3 * index : 3 * index + 3]
+        reads.append(
+            _TokenRead(
+                address=address,
+                symbol=resolve_symbol(address) or "?",
+                decimals=decimals,
+                balance=balance,
+                price_usd=price.readable() if price else None,
             )
-        reads: list[_TokenRead] = []
-        for address, symbol, decimals, balance, price in futures:
-            price_value = price.result()
-            reads.append(
-                _TokenRead(
-                    address=address,
-                    symbol=symbol.result() or "?",
-                    decimals=decimals.result(),
-                    balance=balance.result(),
-                    price_usd=price_value.readable() if price_value else None,
-                )
-            )
-        return reads
+        )
+    return reads
 
 
 def _value_tokens(

{ipor_fusion-4.0.0.dev114 → ipor_fusion-4.0.0.dev118}/tests/conftest.py

@@ -49,6 +49,7 @@ def sequential_multicall():
         patch("ipor_fusion.vault_reads.Multicall3", SequentialMulticall),
         patch("ipor_fusion.vault_positions.Multicall3", SequentialMulticall),
         patch("ipor_fusion.vault_data.Multicall3", SequentialMulticall),
+        patch("ipor_fusion.vault_erc20.Multicall3", SequentialMulticall),
     ):
         yield
 

{ipor_fusion-4.0.0.dev114 → ipor_fusion-4.0.0.dev118}/tests/test_cli_helpers.py

@@ -752,6 +752,7 @@ class FakeBalanceFuse:
     fuse: str
 
 
+@pytest.mark.usefixtures("sequential_multicall")
 class TestPrintErc20Balances:
     @pytest.fixture(autouse=True)
     def _tmp_config(self, tmp_path, monkeypatch):
@@ -1227,6 +1228,7 @@ class TestPrintReconciliation:
         assert "MISMATCH" in captured.out
 
 
+@pytest.mark.usefixtures("sequential_multicall")
 class TestErc20BalancesNotes:
     @pytest.fixture(autouse=True)
     def _tmp_config(self, tmp_path, monkeypatch):

{ipor_fusion-4.0.0.dev114 → ipor_fusion-4.0.0.dev118}/tests/test_web3_context.py

@@ -1,6 +1,10 @@
 # pyright: reportAttributeAccessIssue=false
 """Unit tests for Web3Context — mock Web3 calls, no network required."""
 
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any
 from unittest.mock import MagicMock, patch
 
 import pytest
@@ -126,7 +130,7 @@ class TestFromUrl:
 
         def make_request(_provider, method, _params):
             methods.append(method)
-            result = "0x1" if method == "eth_chainId" else "0x" + "00" * 32
+            result = "0x1" + "00" * 32
             return {"jsonrpc": "2.0", "id": 1, "result": result}
 
         with patch.object(HTTPProvider, "make_request", make_request):
@@ -635,3 +639,139 @@ class TestPinnedContext:
         assert ctx.call.call_args.args == (ADDR, b"\x01")
         call.call(ctx, block=55)
         assert ctx.call.call_args.args == (ADDR, b"\x01", 55)
+
+
+class TestConcurrencyLimit:
+    def _peak(self, limit: int | None) -> int:
+        in_flight = 0
+        peak = 0
+        lock = threading.Lock()
+
+        def make_request(provider, method, params):
+            nonlocal in_flight, peak
+            with lock:
+                in_flight += 1
+                peak = max(peak, in_flight)
+            time.sleep(0.02)
+            with lock:
+                in_flight -= 1
+            result = "0x1"
+            return {"jsonrpc": "2.0", "id": 1, "result": result}
+
+        with patch.object(HTTPProvider, "make_request", make_request):
+            ctx = Web3Context.from_url(
+                "http://localhost:8545", max_concurrent_requests=limit
+            )
+            with ThreadPoolExecutor(16) as pool:
+                list(pool.map(lambda _: ctx.web3.eth.block_number, range(32)))
+        return peak
+
+    def test_requests_in_flight_never_exceed_the_limit(self):
+        assert self._peak(3) <= 3
+
+    def test_none_leaves_requests_unbounded(self):
+        assert self._peak(None) > 3
+
+    def test_a_nested_middleware_request_does_not_deadlock_at_limit_one(self):
+        """web3's gas price strategy reads a block while handling
+        eth_sendTransaction; that nested request must not wait on the slot the
+        outer one would hold."""
+        block = {
+            "number": "0x1",
+            "hash": "0x" + "11" * 32,
+            "parentHash": "0x" + "22" * 32,
+            "timestamp": "0x1",
+            "baseFeePerGas": "0x1",
+            "transactions": [],
+        }
+
+        def make_request(provider, method, params):
+            result: Any = {
+                "eth_chainId": "0x1",
+                "eth_getBlockByNumber": block,
+                "eth_sendTransaction": "0x" + "ab" * 32,
+            }.get(method, "0x1")
+            return {"jsonrpc": "2.0", "id": 1, "result": result}
+
+        with patch.object(HTTPProvider, "make_request", make_request):
+            ctx = Web3Context.from_url(
+                "http://localhost:8545", max_concurrent_requests=1
+            )
+            done = threading.Event()
+
+            def send():
+                ctx.web3.eth.send_transaction(
+                    {"from": ADDR, "to": ADDR, "gas": 21_000, "gasPrice": 1}
+                )
+                done.set()
+
+            worker = threading.Thread(target=send, daemon=True)
+            worker.start()
+            worker.join(timeout=2)
+        assert done.is_set()
+
+    def _batch_peak(self, limit: int) -> int:
+        in_flight = 0
+        peak = 0
+        lock = threading.Lock()
+
+        def make_request(provider, method, params):
+            return {"jsonrpc": "2.0", "id": 1, "result": "0x1"}
+
+        def make_batch_request(provider, requests):
+            nonlocal in_flight, peak
+            with lock:
+                in_flight += 1
+                peak = max(peak, in_flight)
+            time.sleep(0.02)
+            with lock:
+                in_flight -= 1
+            return [
+                {"jsonrpc": "2.0", "id": i, "result": "0x1"}
+                for i, _ in enumerate(requests)
+            ]
+
+        with (
+            patch.object(HTTPProvider, "make_request", make_request),
+            patch.object(HTTPProvider, "make_batch_request", make_batch_request),
+        ):
+            ctx = Web3Context.from_url(
+                "http://localhost:8545", max_concurrent_requests=limit
+            )
+
+            def one_batch(_):
+                with ctx.web3.batch_requests() as batch:
+                    batch.add(ctx.web3.eth.get_block_number())
+                    return batch.execute()
+
+            with ThreadPoolExecutor(8) as pool:
+                list(pool.map(one_batch, range(16)))
+        return peak
+
+    def test_batches_take_one_slot_each_within_the_limit(self):
+        assert self._batch_peak(2) <= 2
+
+    def test_a_failed_request_releases_its_slot(self):
+        calls = 0
+
+        def make_request(provider, method, params):
+            nonlocal calls
+            calls += 1
+            if calls == 2:
+                raise ConnectionError("dropped")
+            return {"jsonrpc": "2.0", "id": 1, "result": "0x1"}
+
+        with patch.object(HTTPProvider, "make_request", make_request):
+            ctx = Web3Context.from_url(
+                "http://localhost:8545", max_concurrent_requests=1
+            )
+            with pytest.raises(ConnectionError):
+                ctx.web3.eth.block_number  # noqa: B018
+            assert ctx.web3.eth.block_number == 1
+
+    def test_a_limit_below_one_is_rejected(self):
+        with (
+            patch.object(HTTPProvider, "make_request"),
+            pytest.raises(ValueError, match="at least 1"),
+        ):
+            Web3Context.from_url("http://localhost:8545", max_concurrent_requests=0)