iwa 0.0.60__tar.gz → 0.0.62__tar.gz

{iwa-0.0.60/src/iwa.egg-info → iwa-0.0.62}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iwa
-Version: 0.0.60
+Version: 0.0.62
 Summary: A secure, modular, and plugin-based framework for crypto agents and ops
 Requires-Python: <4.0,>=3.12
 Description-Content-Type: text/markdown

{iwa-0.0.60 → iwa-0.0.62}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "iwa"
-version = "0.0.60"
+version = "0.0.62"
 description = "A secure, modular, and plugin-based framework for crypto agents and ops"
 readme = "README.md"
 requires-python = ">=3.12,<4.0"
@@ -72,7 +72,7 @@ where = ["src"]
 
 [tool.ruff]
 line-length = 100
-target-version = "0.0.60"
+target-version = "0.0.62"
 fix = true
 
 [tool.ruff.lint]

{iwa-0.0.60 → iwa-0.0.62}/src/iwa/core/chain/interface.py

@@ -54,9 +54,41 @@ class ChainInterface:
 
         self._initial_block = 0
         self._rotation_lock = threading.Lock()
-        self._session = requests.Session()
+        self._session = self._create_session()
+
+        # Enrich with public RPCs from ChainList (skip for Tenderly vNets)
+        if not self.is_tenderly:
+            self._enrich_rpcs_from_chainlist()
+
         self._init_web3()
 
+    def _create_session(self) -> requests.Session:
+        """Create a requests Session with bounded connection pooling.
+
+        Configures the session with limited pool sizes to prevent file
+        descriptor exhaustion during RPC rotations. Connections are reused
+        within the pool but won't accumulate unboundedly.
+        """
+        session = requests.Session()
+        # Limit pool size: we only talk to one RPC at a time, but may rotate
+        # through multiple during the session lifetime. Keep modest limits.
+        adapter = requests.adapters.HTTPAdapter(
+            pool_connections=5,  # Max different hosts to keep connections to
+            pool_maxsize=10,  # Max connections per host
+        )
+        session.mount("https://", adapter)
+        session.mount("http://", adapter)
+        return session
+
+    def close(self) -> None:
+        """Close the session and release all connections.
+
+        Call this when the ChainInterface is no longer needed to ensure
+        proper cleanup of network resources.
+        """
+        if hasattr(self, "_session") and self._session:
+            self._session.close()
+
     @property
     def current_rpc(self) -> str:
         """Get the current active RPC URL."""
@@ -251,6 +283,39 @@ class ChainInterface:
         ]
         return any(signal in err_text for signal in quota_signals)
 
+    # -- ChainList enrichment ----------------------------------------------
+
+    MAX_RPCS = 20  # Cap total RPCs per chain
+
+    def _enrich_rpcs_from_chainlist(self) -> None:
+        """Add validated public RPCs from ChainList to the rotation pool."""
+        if len(self.chain.rpcs) >= self.MAX_RPCS:
+            logger.debug(
+                f"{self.chain.name}: skipping ChainList enrichment "
+                f"(already have {len(self.chain.rpcs)} RPCs)"
+            )
+            return
+
+        try:
+            from iwa.core.chainlist import ChainlistRPC
+
+            chainlist = ChainlistRPC()
+            extra = chainlist.get_validated_rpcs(
+                self.chain.chain_id,
+                existing_rpcs=self.chain.rpcs,
+                max_results=self.MAX_RPCS - len(self.chain.rpcs),
+            )
+            if extra:
+                self.chain.rpcs.extend(extra)
+                logger.info(
+                    f"Enriched {self.chain.name} with {len(extra)} "
+                    f"ChainList RPCs (total: {len(self.chain.rpcs)})"
+                )
+        except Exception as e:
+            logger.debug(
+                f"ChainList enrichment failed for {self.chain.name}: {e}"
+            )
+
     # -- Per-RPC health tracking ------------------------------------------
 
     def _mark_rpc_backoff(self, index: int, seconds: float) -> None:
@@ -293,35 +358,45 @@ class ChainInterface:
 
         if should_rotate:
             failed_index = self._current_rpc_index
+            failed_rpc = sanitize_rpc_url(self.chain.rpcs[failed_index]) if self.chain.rpcs else "?"
 
             # Apply per-RPC backoff so smart rotation skips this RPC.
             if result["is_quota_exceeded"]:
-                error_type = "quota exceeded"
-                self._mark_rpc_backoff(failed_index, self.QUOTA_EXCEEDED_BACKOFF)
+                error_type = "QUOTA"
+                backoff = self.QUOTA_EXCEEDED_BACKOFF
+                self._mark_rpc_backoff(failed_index, backoff)
             elif result["is_rate_limit"]:
-                error_type = "rate limit"
-                self._mark_rpc_backoff(failed_index, self.RATE_LIMIT_BACKOFF)
+                error_type = "RATE_LIMIT"
+                backoff = self.RATE_LIMIT_BACKOFF
+                self._mark_rpc_backoff(failed_index, backoff)
                 # Brief global backoff so other threads don't immediately flood
                 # the same (now backed-off) RPC before rotation takes effect.
                 self._rate_limiter.trigger_backoff(seconds=2.0)
             else:
-                error_type = "connection"
-                self._mark_rpc_backoff(failed_index, self.CONNECTION_ERROR_BACKOFF)
+                error_type = "CONNECTION"
+                backoff = self.CONNECTION_ERROR_BACKOFF
+                self._mark_rpc_backoff(failed_index, backoff)
+
+            # Count healthy RPCs for visibility
+            healthy_count = sum(1 for i in range(len(self.chain.rpcs)) if self._is_rpc_healthy(i))
+            total_rpcs = len(self.chain.rpcs) if self.chain.rpcs else 0
 
             logger.warning(
-                f"RPC {error_type} error on {self.chain.name} "
-                f"(RPC #{failed_index}): {error}"
+                f"[{self.chain.name}] RPC #{failed_index} {error_type} → "
+                f"backoff {int(backoff)}s ({healthy_count}/{total_rpcs} healthy) | "
+                f"{failed_rpc}: {str(error)[:100]}"
             )
 
             if self.rotate_rpc():
                 result["rotated"] = True
                 result["should_retry"] = True
-                logger.info(f"Rotated to RPC #{self._current_rpc_index} for {self.chain.name}")
+                new_rpc = sanitize_rpc_url(self.chain.rpcs[self._current_rpc_index])
+                logger.info(f"[{self.chain.name}] Rotated to RPC #{self._current_rpc_index}: {new_rpc}")
             else:
                 # Rotation skipped (cooldown or single RPC) - still allow retry
                 result["should_retry"] = True
-                logger.info(
-                    f"RPC rotation skipped, retrying with current RPC #{self._current_rpc_index}"
+                logger.debug(
+                    f"[{self.chain.name}] Rotation skipped (cooldown), retrying RPC #{self._current_rpc_index}"
                 )
 
         elif result["is_server_error"]:

{iwa-0.0.60 → iwa-0.0.62}/src/iwa/core/chain/manager.py

@@ -36,3 +36,11 @@ class ChainInterfaces:
         for name, interface in self.items():
             results[name] = interface.check_rpc_health()
         return results
+
+    def close_all(self) -> None:
+        """Close all chain interface sessions.
+
+        Call this at application shutdown to release network resources.
+        """
+        for _, interface in self.items():
+            interface.close()

iwa-0.0.62/src/iwa/core/chainlist.py

@@ -0,0 +1,299 @@
+"""Module for fetching and parsing RPCs from Chainlist.org."""
+
+import json
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Tuple
+
+import requests
+
+from iwa.core.constants import CACHE_DIR
+from iwa.core.utils import configure_logger
+
+logger = configure_logger()
+
+# -- RPC probing constants --------------------------------------------------
+
+MAX_CHAINLIST_CANDIDATES = 15  # Probe at most this many candidates
+PROBE_TIMEOUT = 5.0  # Seconds per probe request
+MAX_BLOCK_LAG = 10  # Blocks behind majority → considered stale
+
+
+def _normalize_url(url: str) -> str:
+    """Normalize an RPC URL for deduplication (lowercase, strip trailing slash)."""
+    return url.rstrip("/").lower()
+
+
+def _is_template_url(url: str) -> bool:
+    """Return True if the URL contains template variables requiring an API key."""
+    return "${" in url or "{" in url
+
+
+def probe_rpc(
+    url: str,
+    timeout: float = PROBE_TIMEOUT,
+    session: Optional[requests.Session] = None,
+) -> Optional[Tuple[str, float, int]]:
+    """Probe an RPC endpoint with eth_blockNumber.
+
+    Returns ``(url, latency_ms, block_number)`` on success, or ``None``
+    if the endpoint is unreachable, slow, or returns invalid data.
+
+    Args:
+        url: The RPC endpoint URL to probe.
+        timeout: Request timeout in seconds.
+        session: Optional requests.Session for connection reuse. If None,
+                 creates a temporary session that is properly closed.
+
+    """
+    # Use provided session or create temporary one with proper cleanup
+    own_session = session is None
+    if own_session:
+        session = requests.Session()
+
+    try:
+        start = time.monotonic()
+        resp = session.post(
+            url,
+            json={"jsonrpc": "2.0", "method": "eth_blockNumber", "params": [], "id": 1},
+            timeout=timeout,
+        )
+        latency_ms = (time.monotonic() - start) * 1000
+        data = resp.json()
+        block_hex = data.get("result")
+        if not block_hex or not isinstance(block_hex, str) or block_hex == "0x0":
+            return None
+        return (url, latency_ms, int(block_hex, 16))
+    except Exception:
+        return None
+    finally:
+        if own_session:
+            session.close()
+
+
+def _filter_candidates(
+    nodes: "List[RPCNode]",
+    existing_normalized: set,
+) -> List[str]:
+    """Filter ChainList nodes to usable HTTPS candidates."""
+    candidates: List[str] = []
+    for node in nodes:
+        url = node.url
+        if not url.startswith("https://"):
+            continue
+        if _is_template_url(url):
+            continue
+        if _normalize_url(url) in existing_normalized:
+            continue
+        candidates.append(url)
+        if len(candidates) >= MAX_CHAINLIST_CANDIDATES:
+            break
+    return candidates
+
+
+def _probe_candidates(
+    candidates: List[str],
+) -> List[Tuple[str, float, int]]:
+    """Probe a list of RPC URLs in parallel, returning successful results.
+
+    Uses a shared session for all probes to enable connection pooling and
+    ensure proper cleanup of all connections when probing completes.
+    """
+    results: List[Tuple[str, float, int]] = []
+    # Use a shared session with connection pooling for all probes
+    # This prevents FD leaks from individual probe connections
+    with requests.Session() as session:
+        # Configure connection pool size to match our max workers
+        adapter = requests.adapters.HTTPAdapter(
+            pool_connections=10,
+            pool_maxsize=10,
+            max_retries=0,  # No retries - we handle failure gracefully
+        )
+        session.mount("https://", adapter)
+        session.mount("http://", adapter)
+
+        with ThreadPoolExecutor(max_workers=min(len(candidates), 10)) as pool:
+            futures = {
+                pool.submit(probe_rpc, url, PROBE_TIMEOUT, session): url
+                for url in candidates
+            }
+            for future in as_completed(futures, timeout=15):
+                try:
+                    result = future.result()
+                    if result is not None:
+                        results.append(result)
+                except Exception:
+                    pass
+    # Session is closed here via context manager, releasing all connections
+    return results
+
+
+def _rank_and_select(
+    results: List[Tuple[str, float, int]],
+    candidates: List[str],
+    chain_id: int,
+    max_results: int,
+) -> List[str]:
+    """Rank probed RPCs by latency, filtering stale ones."""
+    blocks = sorted(r[2] for r in results)
+    median_block = blocks[len(blocks) // 2]
+
+    valid = [
+        (url, latency)
+        for url, latency, block in results
+        if median_block - block <= MAX_BLOCK_LAG
+    ]
+    valid.sort(key=lambda x: x[1])
+
+    selected = [url for url, _ in valid[:max_results]]
+    if selected:
+        logger.info(
+            f"ChainList: validated {len(selected)}/{len(candidates)} "
+            f"candidates for chain {chain_id} "
+            f"(median block: {median_block})"
+        )
+    return selected
+
+
+@dataclass
+class RPCNode:
+    """Represents a single RPC node with its properties."""
+
+    url: str
+    is_working: bool
+    privacy: Optional[str] = None
+    tracking: Optional[str] = None
+
+    @property
+    def is_tracking(self) -> bool:
+        """Returns True if the RPC is known to track user data."""
+        return self.privacy == "privacy" or self.tracking in ("limited", "yes")
+
+
+class ChainlistRPC:
+    """Fetcher and parser for Chainlist RPC data."""
+
+    URL = "https://chainlist.org/rpcs.json"
+    CACHE_PATH = CACHE_DIR / "chainlist_rpcs.json"
+    CACHE_TTL = 86400  # 24 hours
+
+    def __init__(self) -> None:
+        """Initialize the ChainlistRPC instance."""
+        self._data: List[Dict[str, Any]] = []
+
+    def fetch_data(self, force_refresh: bool = False) -> None:
+        """Fetches the RPC data from Chainlist with local caching."""
+        # 1. Try local cache first unless force_refresh is requested
+        if not force_refresh and self.CACHE_PATH.exists():
+            try:
+                mtime = self.CACHE_PATH.stat().st_mtime
+                if time.time() - mtime < self.CACHE_TTL:
+                    with self.CACHE_PATH.open("r") as f:
+                        self._data = json.load(f)
+                    if self._data:
+                        return
+            except Exception as e:
+                print(f"Error reading Chainlist cache: {e}")
+
+        # 2. Fetch from remote (use session context for proper cleanup)
+        try:
+            with requests.Session() as session:
+                response = session.get(self.URL, timeout=10)
+                response.raise_for_status()
+                self._data = response.json()
+
+            # 3. Update local cache
+            if self._data:
+                self.CACHE_PATH.parent.mkdir(parents=True, exist_ok=True)
+                with self.CACHE_PATH.open("w") as f:
+                    json.dump(self._data, f)
+        except requests.RequestException as e:
+            print(f"Error fetching Chainlist data from {self.URL}: {e}")
+            # Fallback to expired cache if available
+            if not self._data and self.CACHE_PATH.exists():
+                try:
+                    with self.CACHE_PATH.open("r") as f:
+                        self._data = json.load(f)
+                except Exception:
+                    pass
+            if not self._data:
+                self._data = []
+
+    def get_chain_data(self, chain_id: int) -> Optional[Dict[str, Any]]:
+        """Returns the raw chain data for a specific chain ID."""
+        if not self._data:
+            self.fetch_data()
+
+        for entry in self._data:
+            if entry.get("chainId") == chain_id:
+                return entry
+        return None
+
+    def get_rpcs(self, chain_id: int) -> List[RPCNode]:
+        """Returns a list of RPCNode objects for a parsed and cleaner view."""
+        chain_data = self.get_chain_data(chain_id)
+        if not chain_data:
+            return []
+
+        raw_rpcs = chain_data.get("rpc", [])
+        nodes = []
+        for rpc in raw_rpcs:
+            nodes.append(
+                RPCNode(
+                    url=rpc.get("url", ""),
+                    is_working=True,
+                    privacy=rpc.get("privacy"),
+                    tracking=rpc.get("tracking"),
+                )
+            )
+        return nodes
+
+    def get_https_rpcs(self, chain_id: int) -> List[str]:
+        """Returns a list of HTTPS RPC URLs for the given chain."""
+        rpcs = self.get_rpcs(chain_id)
+        return [
+            node.url
+            for node in rpcs
+            if node.url.startswith("https://") or node.url.startswith("http://")
+        ]
+
+    def get_wss_rpcs(self, chain_id: int) -> List[str]:
+        """Returns a list of WSS RPC URLs for the given chain."""
+        rpcs = self.get_rpcs(chain_id)
+        return [
+            node.url
+            for node in rpcs
+            if node.url.startswith("wss://") or node.url.startswith("ws://")
+        ]
+
+    def get_validated_rpcs(
+        self,
+        chain_id: int,
+        existing_rpcs: List[str],
+        max_results: int = 5,
+    ) -> List[str]:
+        """Return ChainList RPCs filtered, probed, and sorted by quality.
+
+        1. Fetch HTTPS RPCs from ChainList for *chain_id*.
+        2. Filter out template URLs, duplicates of *existing_rpcs*, and
+           websocket endpoints.
+        3. Probe the top candidates in parallel with ``eth_blockNumber``.
+        4. Discard RPCs that are stale (block number lagging behind majority).
+        5. Return up to *max_results* URLs sorted by latency (fastest first).
+        """
+        nodes = self.get_rpcs(chain_id)
+        if not nodes:
+            return []
+
+        existing_normalized = {_normalize_url(u) for u in existing_rpcs}
+        candidates = _filter_candidates(nodes, existing_normalized)
+        if not candidates:
+            return []
+
+        results = _probe_candidates(candidates)
+        if not results:
+            return []
+
+        selected = _rank_and_select(results, candidates, chain_id, max_results)
+        return selected