csm-dashboard 0.2.0__py3-none-any.whl

src/data/ipfs_logs.py

@@ -0,0 +1,267 @@
+"""IPFS distribution log fetching with persistent caching."""
+
+import json
+import time
+from dataclasses import dataclass
+from decimal import Decimal
+from pathlib import Path
+
+import httpx
+
+from ..core.config import get_settings
+
+
+@dataclass
+class FrameData:
+    """Data from a single distribution frame."""
+
+    start_epoch: int
+    end_epoch: int
+    log_cid: str
+    block_number: int
+    distributed_rewards: int  # For specific operator, in wei
+
+
+class IPFSLogProvider:
+    """Fetches and caches historical distribution logs from IPFS."""
+
+    # IPFS gateways to try in order
+    GATEWAYS = [
+        "https://dweb.link/ipfs/",
+        "https://ipfs.io/ipfs/",
+        "https://cloudflare-ipfs.com/ipfs/",
+    ]
+
+    # Rate limiting: minimum seconds between gateway requests
+    MIN_REQUEST_INTERVAL = 1.0
+
+    def __init__(self, cache_dir: Path | None = None):
+        self.settings = get_settings()
+        self.cache_dir = cache_dir or Path.home() / ".cache" / "csm-dashboard" / "ipfs"
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self._last_request_time = 0.0
+
+    def _get_cache_path(self, cid: str) -> Path:
+        """Get the cache file path for a CID."""
+        return self.cache_dir / f"{cid}.json"
+
+    def _load_from_cache(self, cid: str) -> dict | None:
+        """Load log data from local cache if available."""
+        cache_path = self._get_cache_path(cid)
+        if cache_path.exists():
+            try:
+                with open(cache_path) as f:
+                    return json.load(f)
+            except (json.JSONDecodeError, OSError):
+                # Corrupted cache, remove it
+                cache_path.unlink(missing_ok=True)
+        return None
+
+    def _save_to_cache(self, cid: str, data: dict) -> None:
+        """Save log data to local cache."""
+        cache_path = self._get_cache_path(cid)
+        try:
+            with open(cache_path, "w") as f:
+                json.dump(data, f)
+        except OSError:
+            pass  # Cache write failure is non-fatal
+
+    def _rate_limit(self) -> None:
+        """Ensure minimum interval between IPFS gateway requests."""
+        now = time.time()
+        elapsed = now - self._last_request_time
+        if elapsed < self.MIN_REQUEST_INTERVAL:
+            time.sleep(self.MIN_REQUEST_INTERVAL - elapsed)
+        self._last_request_time = time.time()
+
+    async def fetch_log(self, cid: str) -> dict | None:
+        """
+        Fetch and parse a distribution log from IPFS.
+
+        Checks local cache first, then tries IPFS gateways.
+        Returns None if fetch fails.
+        """
+        # Check cache first
+        cached = self._load_from_cache(cid)
+        if cached is not None:
+            return cached
+
+        # Rate limit gateway requests
+        self._rate_limit()
+
+        # Try each gateway
+        async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
+            for gateway in self.GATEWAYS:
+                try:
+                    url = f"{gateway}{cid}"
+                    response = await client.get(url)
+                    if response.status_code == 200:
+                        data = response.json()
+                        # The IPFS log is wrapped in a list, unwrap it
+                        if isinstance(data, list) and len(data) == 1:
+                            data = data[0]
+                        # Cache the successful result
+                        self._save_to_cache(cid, data)
+                        return data
+                except Exception:
+                    continue  # Try next gateway
+
+        return None
+
+    def get_operator_frame_rewards(self, log_data: dict, operator_id: int) -> int | None:
+        """
+        Extract operator's distributed_rewards for a frame.
+
+        Returns rewards in wei (shares), or None if operator not in frame.
+        """
+        operators = log_data.get("operators", {})
+        op_key = str(operator_id)
+
+        if op_key not in operators:
+            return None
+
+        return operators[op_key].get("distributed_rewards", 0)
+
+    def get_frame_info(self, log_data: dict) -> tuple[int, int]:
+        """
+        Extract frame epoch range from log data.
+
+        Returns (start_epoch, end_epoch).
+        """
+        frame = log_data.get("frame", [0, 0])
+        if not isinstance(frame, list) or len(frame) < 2:
+            return (0, 0)
+        return (frame[0], frame[1])
+
+    async def get_operator_history(
+        self,
+        operator_id: int,
+        log_cids: list[dict],  # List of {block, logCid} from events
+    ) -> list[FrameData]:
+        """
+        Fetch all historical frame data for an operator.
+
+        Args:
+            operator_id: The operator ID to look up
+            log_cids: List of {block, logCid} dicts from DistributionLogUpdated events
+
+        Returns:
+            List of FrameData objects, sorted by epoch (oldest first)
+        """
+        frames = []
+
+        for entry in log_cids:
+            cid = entry["logCid"]
+            block = entry["block"]
+
+            log_data = await self.fetch_log(cid)
+            if log_data is None:
+                continue
+
+            rewards = self.get_operator_frame_rewards(log_data, operator_id)
+            if rewards is None:
+                # Operator not in this frame (may have joined later)
+                continue
+
+            start_epoch, end_epoch = self.get_frame_info(log_data)
+
+            frames.append(
+                FrameData(
+                    start_epoch=start_epoch,
+                    end_epoch=end_epoch,
+                    log_cid=cid,
+                    block_number=block,
+                    distributed_rewards=rewards,
+                )
+            )
+
+        # Sort by epoch (oldest first)
+        frames.sort(key=lambda f: f.start_epoch)
+        return frames
+
+    def calculate_frame_duration_days(self, frame: FrameData) -> float:
+        """Calculate the duration of a frame in days."""
+        # Each epoch is 6.4 minutes (384 seconds = 32 slots * 12 seconds)
+        epochs = frame.end_epoch - frame.start_epoch
+        minutes = epochs * 6.4
+        return minutes / (60 * 24)
+
+    def calculate_historical_apy(
+        self,
+        frames: list[FrameData],
+        bond_eth: Decimal,
+        periods: list[int] | None = None,
+    ) -> dict[str, float | None]:
+        """
+        Calculate APY from historical frame data.
+
+        Args:
+            frames: List of FrameData objects (oldest first)
+            bond_eth: Current bond in ETH (used for all periods)
+            periods: List of day counts to calculate APY for (default: [28, None] for 28d and LTD)
+
+        Returns:
+            Dict mapping period name to APY percentage (e.g., {"28d": 3.92, "ltd": 4.10})
+
+        Note:
+            For lifetime APY, only frames with non-zero rewards are included. This avoids
+            artificially low APY for operators who had a ramp-up period with no rewards.
+        """
+        if periods is None:
+            periods = [28, None]  # 28-day and lifetime
+
+        if not frames or bond_eth <= 0:
+            return {self._period_name(p): None for p in periods}
+
+        results = {}
+
+        for period in periods:
+            if period is None:
+                # Lifetime: only frames where operator earned rewards
+                # This avoids ramp-up periods with 0 rewards skewing the APY
+                selected_frames = [f for f in frames if f.distributed_rewards > 0]
+            else:
+                # Select frames within the period
+                # Work backwards from most recent frame
+                total_days = 0.0
+                selected_frames = []
+                for frame in reversed(frames):
+                    frame_days = self.calculate_frame_duration_days(frame)
+                    if total_days + frame_days <= period * 1.5:  # Allow some buffer
+                        selected_frames.insert(0, frame)
+                        total_days += frame_days
+                    if total_days >= period:
+                        break
+
+            if not selected_frames:
+                results[self._period_name(period)] = None
+                continue
+
+            # Sum rewards and calculate total days
+            total_rewards_wei = sum(f.distributed_rewards for f in selected_frames)
+            total_days = sum(self.calculate_frame_duration_days(f) for f in selected_frames)
+
+            if total_days <= 0:
+                results[self._period_name(period)] = None
+                continue
+
+            # Convert rewards to ETH
+            total_rewards_eth = Decimal(total_rewards_wei) / Decimal(10**18)
+
+            # Annualize: (rewards / bond) * (365 / days) * 100
+            apy = float(total_rewards_eth / bond_eth) * (365.0 / total_days) * 100
+
+            results[self._period_name(period)] = round(apy, 2)
+
+        return results
+
+    def _period_name(self, period: int | None) -> str:
+        """Convert period days to display name."""
+        if period is None:
+            return "ltd"
+        return f"{period}d"
+
+    def clear_cache(self) -> None:
+        """Clear all cached IPFS logs."""
+        for cache_file in self.cache_dir.glob("*.json"):
+            cache_file.unlink(missing_ok=True)

src/data/known_cids.py

@@ -0,0 +1,30 @@
+"""Known historical distribution log CIDs as fallback.
+
+These are manually curated from on-chain DistributionLogUpdated events.
+Used when neither Etherscan API nor RPC event queries are available.
+
+To update this list:
+1. Query events from Etherscan or a full node
+2. Add new entries in chronological order (oldest first)
+
+Last updated: December 2025
+"""
+
+# Format: {"block": block_number, "logCid": "Qm..."}
+KNOWN_DISTRIBUTION_LOGS: list[dict] = [
+    # CSM launched Dec 2024, first distribution frame ~Jan 2025
+    {"block": 21277898, "logCid": "QmezkGCHPUJ9XSAJfibmo6Sup35VgbhnodfYsc1xNT3rbo"},
+    {"block": 21445874, "logCid": "Qmb5CZUD9uLXP9LS68jnJp1v2GTF1KjYsNLJuML9fpRufE"},
+    {"block": 21644860, "logCid": "QmePUqG8tMXbv3eHDu3j56Dod4gwmGh1Vapsh7u4gxotT4"},
+    {"block": 21859279, "logCid": "QmT5JWn3sR7fYxxxSh3kHBmjZyPBWjKb6CsSLGXMQbLXMX"},
+    {"block": 22047254, "logCid": "QmWxANi2GWvoxwnPRsxwZNF6NRyjwMBPAF4bBMcL3HGG3i"},
+    {"block": 22247841, "logCid": "QmYQPDuqVbxWq2YNSZS55LE3eTeriy51HTCgBHLiW9fN7N"},
+    {"block": 22448016, "logCid": "QmeZduNqrnSMLTVE5tkNDv2WhtL3uAgHRP1915m5CHcCqM"},
+    {"block": 22646060, "logCid": "QmaHU6Ah99Yk6kQVtSrN4inxqqYoU6epZ5UKyDvwdYUKAS"},
+    {"block": 22847998, "logCid": "Qmemm9gD2fQgwNziBsf9mAaveNXJ3eJvHpqBTWKoLdUXXV"},
+    {"block": 23048383, "logCid": "QmVgGQS7QBeRMq2noqqxekY5ezmqRsgu7JjiyMyRaaWEDv"},
+    {"block": 23248929, "logCid": "QmaUC2HBv88mJ9Gf99hfNgtH4qo2F1yHaBMC4imwVhxDDi"},
+    {"block": 23463926, "logCid": "QmPPFkydgtnwMBDF6nZZaU5nnqy3csbKts3UfRRgWXreEu"},
+    {"block": 23649468, "logCid": "QmSdx8WFnaeMWLKURBYgMMiixZ9z4xn3mvGPBeuRzPt6MQ"},
+    {"block": 23849500, "logCid": "QmZyzTYdSait7BYCEToFJFJ6qVkX2HJBrrvXhk64e82xoK"},
+]

src/data/lido_api.py

@@ -0,0 +1,35 @@
+"""Lido protocol API for stETH APR and other metrics."""
+
+import httpx
+
+from .cache import cached
+
+LIDO_API_BASE = "https://eth-api.lido.fi/v1"
+
+
+class LidoAPIProvider:
+    """Fetches data from Lido's public API."""
+
+    @cached(ttl=3600)  # Cache for 1 hour
+    async def get_steth_apr(self) -> dict:
+        """
+        Get current stETH APR from Lido API.
+
+        Returns 7-day SMA (simple moving average) APR.
+        """
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            try:
+                response = await client.get(
+                    f"{LIDO_API_BASE}/protocol/steth/apr/sma"
+                )
+
+                if response.status_code == 200:
+                    data = response.json()
+                    return {
+                        "apr": float(data.get("data", {}).get("smaApr", 0)),
+                        "timestamp": data.get("data", {}).get("timeUnix"),
+                    }
+            except Exception:
+                pass
+
+        return {"apr": None, "timestamp": None}

src/data/onchain.py

@@ -0,0 +1,258 @@
+"""On-chain data fetching via Web3."""
+
+import asyncio
+from decimal import Decimal
+
+from web3 import Web3
+
+from ..core.config import get_settings
+from ..core.contracts import (
+    CSACCOUNTING_ABI,
+    CSFEEDISTRIBUTOR_ABI,
+    CSMODULE_ABI,
+    STETH_ABI,
+)
+from ..core.types import BondSummary, NodeOperator
+from .cache import cached
+from .etherscan import EtherscanProvider
+from .known_cids import KNOWN_DISTRIBUTION_LOGS
+
+
+class OnChainDataProvider:
+    """Fetches data from Ethereum contracts."""
+
+    def __init__(self, rpc_url: str | None = None):
+        self.settings = get_settings()
+        self.w3 = Web3(Web3.HTTPProvider(rpc_url or self.settings.eth_rpc_url))
+
+        # Initialize contracts
+        self.csmodule = self.w3.eth.contract(
+            address=Web3.to_checksum_address(self.settings.csmodule_address),
+            abi=CSMODULE_ABI,
+        )
+        self.csaccounting = self.w3.eth.contract(
+            address=Web3.to_checksum_address(self.settings.csaccounting_address),
+            abi=CSACCOUNTING_ABI,
+        )
+        self.csfeedistributor = self.w3.eth.contract(
+            address=Web3.to_checksum_address(self.settings.csfeedistributor_address),
+            abi=CSFEEDISTRIBUTOR_ABI,
+        )
+        self.steth = self.w3.eth.contract(
+            address=Web3.to_checksum_address(self.settings.steth_address),
+            abi=STETH_ABI,
+        )
+
+    @cached(ttl=60)
+    async def get_node_operators_count(self) -> int:
+        """Get total number of node operators."""
+        return self.csmodule.functions.getNodeOperatorsCount().call()
+
+    @cached(ttl=300)
+    async def get_node_operator(self, operator_id: int) -> NodeOperator:
+        """Get node operator data by ID."""
+        data = self.csmodule.functions.getNodeOperator(operator_id).call()
+        return NodeOperator(
+            node_operator_id=operator_id,
+            total_added_keys=data[0],
+            total_withdrawn_keys=data[1],
+            total_deposited_keys=data[2],
+            total_vetted_keys=data[3],
+            stuck_validators_count=data[4],
+            depositable_validators_count=data[5],
+            target_limit=data[6],
+            target_limit_mode=data[7],
+            total_exited_keys=data[8],
+            enqueued_count=data[9],
+            manager_address=data[10],
+            proposed_manager_address=data[11],
+            reward_address=data[12],
+            proposed_reward_address=data[13],
+            extended_manager_permissions=data[14],
+        )
+
+    async def find_operator_by_address(self, address: str) -> int | None:
+        """
+        Find operator ID by manager or reward address.
+
+        Tries batch requests first (faster if RPC supports JSON-RPC batching).
+        Falls back to sequential calls with rate limiting if batch fails.
+        """
+        address = Web3.to_checksum_address(address)
+        total = await self.get_node_operators_count()
+
+        # Try batch requests first (not all RPCs support this)
+        batch_size = 50
+        batch_supported = True
+
+        for start in range(0, total, batch_size):
+            end = min(start + batch_size, total)
+
+            if batch_supported:
+                try:
+                    with self.w3.batch_requests() as batch:
+                        for op_id in range(start, end):
+                            batch.add(self.csmodule.functions.getNodeOperator(op_id))
+                        results = batch.execute()
+
+                    for i, data in enumerate(results):
+                        op_id = start + i
+                        manager = data[10]
+                        reward = data[12]
+                        if manager.lower() == address.lower() or reward.lower() == address.lower():
+                            return op_id
+                    continue  # Batch succeeded, move to next batch
+                except Exception:
+                    # Batch not supported by this RPC, fall back to sequential
+                    batch_supported = False
+
+            # Sequential fallback with rate limiting
+            for op_id in range(start, end):
+                try:
+                    data = self.csmodule.functions.getNodeOperator(op_id).call()
+                    manager = data[10]
+                    reward = data[12]
+                    if manager.lower() == address.lower() or reward.lower() == address.lower():
+                        return op_id
+                    # Small delay to avoid rate limiting on public RPCs
+                    await asyncio.sleep(0.05)
+                except Exception:
+                    await asyncio.sleep(0.1)  # Longer delay on error
+                    continue
+
+        return None
+
+    @cached(ttl=60)
+    async def get_bond_summary(self, operator_id: int) -> BondSummary:
+        """Get bond summary for an operator."""
+        current, required = self.csaccounting.functions.getBondSummary(
+            operator_id
+        ).call()
+
+        current_eth = Decimal(current) / Decimal(10**18)
+        required_eth = Decimal(required) / Decimal(10**18)
+        excess_eth = max(Decimal(0), current_eth - required_eth)
+
+        return BondSummary(
+            current_bond_wei=current,
+            required_bond_wei=required,
+            current_bond_eth=current_eth,
+            required_bond_eth=required_eth,
+            excess_bond_eth=excess_eth,
+        )
+
+    @cached(ttl=60)
+    async def get_distributed_shares(self, operator_id: int) -> int:
+        """Get already distributed (claimed) shares for operator."""
+        return self.csfeedistributor.functions.distributedShares(operator_id).call()
+
+    @cached(ttl=60)
+    async def shares_to_eth(self, shares: int) -> Decimal:
+        """Convert stETH shares to ETH value."""
+        if shares == 0:
+            return Decimal(0)
+        eth_wei = self.steth.functions.getPooledEthByShares(shares).call()
+        return Decimal(eth_wei) / Decimal(10**18)
+
+    async def get_signing_keys(
+        self, operator_id: int, start: int = 0, count: int = 100
+    ) -> list[str]:
+        """Get validator pubkeys for an operator.
+
+        Fetches in batches of 100 to avoid RPC limits on large operators.
+        """
+        keys = []
+        batch_size = 100
+
+        for batch_start in range(start, start + count, batch_size):
+            batch_count = min(batch_size, start + count - batch_start)
+            keys_bytes = self.csmodule.functions.getSigningKeys(
+                operator_id, batch_start, batch_count
+            ).call()
+            # Each key is 48 bytes
+            for i in range(0, len(keys_bytes), 48):
+                key = "0x" + keys_bytes[i : i + 48].hex()
+                keys.append(key)
+
+        return keys
+
+    def get_current_log_cid(self) -> str:
+        """Get the current distribution log CID from the contract."""
+        return self.csfeedistributor.functions.logCid().call()
+
+    @cached(ttl=3600)  # Cache for 1 hour since historical events don't change
+    async def get_distribution_log_history(
+        self, start_block: int | None = None
+    ) -> list[dict]:
+        """
+        Query DistributionLogUpdated events to get historical logCids.
+
+        Tries multiple methods in order:
+        1. Etherscan API (if API key configured) - most reliable
+        2. Chunked RPC queries (10k block chunks) - works on some RPCs
+        3. Hardcoded known CIDs - fallback for users without API keys
+        4. Current logCid from contract - ultimate fallback
+
+        Args:
+            start_block: Starting block number (default: CSM deployment ~20873000)
+
+        Returns:
+            List of {block, logCid} dicts, sorted by block number (oldest first)
+        """
+        # CSM was deployed around block 20873000 (Dec 2024)
+        if start_block is None:
+            start_block = 20873000
+
+        # 1. Try Etherscan API first (most reliable)
+        etherscan = EtherscanProvider()
+        if etherscan.is_available():
+            events = await etherscan.get_distribution_log_events(
+                self.settings.csfeedistributor_address,
+                start_block,
+            )
+            if events:
+                return events
+
+        # 2. Try chunked RPC queries
+        events = await self._query_events_chunked(start_block)
+        if events:
+            return events
+
+        # 3. Use known historical CIDs as fallback
+        if KNOWN_DISTRIBUTION_LOGS:
+            return KNOWN_DISTRIBUTION_LOGS
+
+        # 4. Ultimate fallback: current logCid only
+        try:
+            current_cid = self.get_current_log_cid()
+            if current_cid:
+                current_block = self.w3.eth.block_number
+                return [{"block": current_block, "logCid": current_cid}]
+        except Exception:
+            pass
+
+        return []
+
+    async def _query_events_chunked(
+        self, start_block: int, chunk_size: int = 10000
+    ) -> list[dict]:
+        """Query events in smaller chunks to work around RPC limitations."""
+        current_block = self.w3.eth.block_number
+        all_events = []
+
+        for from_block in range(start_block, current_block, chunk_size):
+            to_block = min(from_block + chunk_size - 1, current_block)
+            try:
+                events = self.csfeedistributor.events.DistributionLogUpdated.get_logs(
+                    from_block=from_block,
+                    to_block=to_block,
+                )
+                for e in events:
+                    all_events.append(
+                        {"block": e["blockNumber"], "logCid": e["args"]["logCid"]}
+                    )
+            except Exception:
+                # If chunked queries fail, give up on this method
+                return []
+
+        return sorted(all_events, key=lambda x: x["block"])

src/data/rewards_tree.py

@@ -0,0 +1,58 @@
+"""Fetch and parse the rewards merkle tree from GitHub."""
+
+import httpx
+
+from ..core.config import get_settings
+from ..core.types import RewardsInfo
+from .cache import cached
+
+
+class RewardsTreeProvider:
+    """Fetches rewards data from the csm-rewards repository."""
+
+    def __init__(self):
+        self.settings = get_settings()
+
+    @cached(ttl=3600)  # Cache for 1 hour since tree updates infrequently
+    async def fetch_rewards_data(self) -> dict:
+        """
+        Fetch the proofs.json file which contains:
+        {
+            "CSM Operator 0": {
+                "cumulativeFeeShares": 304687403773285400,
+                "proof": ["0x...", "0x...", ...]
+            },
+            ...
+        }
+        """
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            response = await client.get(self.settings.rewards_proofs_url)
+            response.raise_for_status()
+            return response.json()
+
+    async def get_operator_rewards(self, operator_id: int) -> RewardsInfo | None:
+        """Get rewards info for a specific operator."""
+        data = await self.fetch_rewards_data()
+        key = f"CSM Operator {operator_id}"
+
+        if key not in data:
+            return None
+
+        entry = data[key]
+        return RewardsInfo(
+            cumulative_fee_shares=entry["cumulativeFeeShares"],
+            proof=entry["proof"],
+        )
+
+    async def get_all_operators_with_rewards(self) -> list[int]:
+        """Get list of all operator IDs that have rewards."""
+        data = await self.fetch_rewards_data()
+        operator_ids = []
+        for key in data.keys():
+            if key.startswith("CSM Operator "):
+                try:
+                    op_id = int(key.replace("CSM Operator ", ""))
+                    operator_ids.append(op_id)
+                except ValueError:
+                    continue
+        return sorted(operator_ids)