synapse-filecoin-sdk 0.1.0__py3-none-any.whl

pynapse/retriever/async_chain.py

@@ -0,0 +1,227 @@
+"""
+AsyncChainRetriever - Async queries to on-chain data for piece retrieval.
+
+This retriever uses the Async Warm Storage service to find service providers
+that have the requested piece, then attempts to download from them.
+"""
+from __future__ import annotations
+
+import asyncio
+from typing import List, Optional, Protocol, TYPE_CHECKING
+
+import httpx
+
+if TYPE_CHECKING:
+    from pynapse.sp_registry import AsyncSPRegistryService, ProviderInfo
+    from pynapse.warm_storage import AsyncWarmStorageService
+
+
+class AsyncPieceRetriever(Protocol):
+    """Protocol for async piece retrieval implementations."""
+    
+    async def fetch_piece(
+        self,
+        piece_cid: str,
+        client_address: str,
+        provider_address: Optional[str] = None,
+    ) -> bytes:
+        """Fetch a piece by CID asynchronously."""
+        ...
+
+
+class AsyncChainRetriever:
+    """
+    Async retriever that queries on-chain data to find providers with the piece.
+    
+    This is the standard async retriever that:
+    1. Looks up the client's datasets to find active providers
+    2. Tries to download from each provider until one succeeds
+    
+    Example:
+        retriever = AsyncChainRetriever(warm_storage, sp_registry)
+        data = await retriever.fetch_piece(piece_cid, client_address)
+    """
+    
+    def __init__(
+        self,
+        warm_storage: "AsyncWarmStorageService",
+        sp_registry: "AsyncSPRegistryService",
+        fallback_retriever: Optional[AsyncPieceRetriever] = None,
+        timeout: float = 30.0,
+    ) -> None:
+        self._warm_storage = warm_storage
+        self._sp_registry = sp_registry
+        self._fallback = fallback_retriever
+        self._timeout = timeout
+
+    async def _get_pdp_endpoint(self, provider_id: int) -> Optional[str]:
+        """Get PDP service URL for a provider."""
+        try:
+            product = await self._sp_registry.get_provider_with_product(provider_id, 1)
+            for i, key in enumerate(product.product.capability_keys):
+                if key == "serviceURL" and i < len(product.product_capability_values):
+                    return product.product_capability_values[i]
+        except Exception:
+            pass
+        return None
+
+    async def _find_providers(
+        self,
+        client_address: str,
+        provider_address: Optional[str] = None,
+    ) -> List["ProviderInfo"]:
+        """Find providers that can serve pieces for a client."""
+        
+        if provider_address is not None:
+            # Direct provider case
+            provider = await self._sp_registry.get_provider_by_address(provider_address)
+            if provider is None:
+                raise ValueError(f"Provider {provider_address} not found in registry")
+            return [provider]
+        
+        # Get client's datasets with details
+        datasets = await self._warm_storage.get_client_data_sets_with_details(client_address)
+        
+        # Filter for live datasets with pieces
+        valid_datasets = [
+            ds for ds in datasets
+            if ds.is_live and ds.active_piece_count > 0
+        ]
+        
+        if not valid_datasets:
+            raise ValueError(f"No active datasets with data found for client {client_address}")
+        
+        # Get unique provider IDs
+        unique_provider_ids = list(set(ds.provider_id for ds in valid_datasets))
+        
+        # Fetch provider info for each concurrently
+        async def get_provider_if_active(pid: int) -> Optional["ProviderInfo"]:
+            try:
+                provider = await self._sp_registry.get_provider(pid)
+                if provider and provider.is_active:
+                    return provider
+            except Exception:
+                pass
+            return None
+        
+        results = await asyncio.gather(*[get_provider_if_active(pid) for pid in unique_provider_ids])
+        providers = [p for p in results if p is not None]
+        
+        if not providers:
+            raise ValueError("No valid providers found")
+        
+        return providers
+
+    async def _try_fetch_from_provider(
+        self,
+        provider: "ProviderInfo",
+        piece_cid: str,
+    ) -> Optional[bytes]:
+        """Try to fetch a piece from a specific provider."""
+        endpoint = await self._get_pdp_endpoint(provider.provider_id)
+        if not endpoint:
+            return None
+        
+        try:
+            async with httpx.AsyncClient(timeout=self._timeout) as client:
+                # First check if provider has the piece
+                find_resp = await client.get(
+                    f"{endpoint.rstrip('/')}/pdp/piece",
+                    params={"pieceCid": piece_cid},
+                )
+                if find_resp.status_code != 200:
+                    return None
+                
+                # Download the piece
+                download_resp = await client.get(
+                    f"{endpoint.rstrip('/')}/pdp/piece/{piece_cid}",
+                )
+                if download_resp.status_code == 200:
+                    return download_resp.content
+            
+        except Exception:
+            pass
+        
+        return None
+
+    async def fetch_piece(
+        self,
+        piece_cid: str,
+        client_address: str,
+        provider_address: Optional[str] = None,
+        parallel: bool = True,
+    ) -> bytes:
+        """
+        Fetch a piece by CID.
+        
+        Args:
+            piece_cid: The piece CID to fetch
+            client_address: The client address to look up datasets for
+            provider_address: Optional specific provider address
+            parallel: Whether to try providers in parallel (default: True)
+            
+        Returns:
+            The piece data as bytes
+            
+        Raises:
+            ValueError: If piece cannot be found
+        """
+        providers = await self._find_providers(client_address, provider_address)
+        
+        if parallel:
+            # Try all providers in parallel, return first success
+            async def try_provider(provider: "ProviderInfo") -> Optional[bytes]:
+                return await self._try_fetch_from_provider(provider, piece_cid)
+            
+            # Use asyncio.as_completed for first success
+            tasks = [asyncio.create_task(try_provider(p)) for p in providers]
+            
+            for completed_task in asyncio.as_completed(tasks):
+                try:
+                    result = await completed_task
+                    if result is not None:
+                        # Cancel remaining tasks
+                        for task in tasks:
+                            task.cancel()
+                        return result
+                except Exception:
+                    continue
+            
+        else:
+            # Try providers sequentially
+            for provider in providers:
+                data = await self._try_fetch_from_provider(provider, piece_cid)
+                if data is not None:
+                    return data
+        
+        # Try fallback if configured
+        if self._fallback is not None:
+            return await self._fallback.fetch_piece(piece_cid, client_address, provider_address)
+        
+        raise ValueError(f"Piece {piece_cid} not found on any provider")
+
+    async def fetch_pieces(
+        self,
+        piece_cids: List[str],
+        client_address: str,
+        provider_address: Optional[str] = None,
+    ) -> List[bytes]:
+        """
+        Fetch multiple pieces concurrently.
+        
+        Args:
+            piece_cids: List of piece CIDs to fetch
+            client_address: The client address to look up datasets for
+            provider_address: Optional specific provider address
+            
+        Returns:
+            List of piece data as bytes (in same order as input CIDs)
+            
+        Raises:
+            ValueError: If any piece cannot be found
+        """
+        tasks = [
+            self.fetch_piece(cid, client_address, provider_address)
+            for cid in piece_cids
+        ]
+        return list(await asyncio.gather(*tasks))

pynapse/retriever/chain.py

@@ -0,0 +1,209 @@
+"""
+ChainRetriever - Queries on-chain data to find and retrieve pieces.
+
+This retriever uses the Warm Storage service to find service providers
+that have the requested piece, then attempts to download from them.
+"""
+from __future__ import annotations
+
+import concurrent.futures
+from typing import List, Optional, Protocol, TYPE_CHECKING
+
+import httpx
+
+if TYPE_CHECKING:
+    from pynapse.sp_registry import ProviderInfo
+    from pynapse.warm_storage import SyncWarmStorageService
+
+
+class PieceRetriever(Protocol):
+    """Protocol for piece retrieval implementations."""
+    
+    def fetch_piece(
+        self,
+        piece_cid: str,
+        client_address: str,
+        provider_address: Optional[str] = None,
+    ) -> bytes:
+        """Fetch a piece by CID."""
+        ...
+
+
+class ChainRetriever:
+    """
+    Retriever that queries on-chain data to find providers with the piece.
+    
+    This is the standard retriever that:
+    1. Looks up the client's datasets to find active providers
+    2. Tries to download from each provider until one succeeds
+    
+    Example:
+        retriever = ChainRetriever(warm_storage, sp_registry)
+        data = retriever.fetch_piece(piece_cid, client_address)
+    """
+    
+    def __init__(
+        self,
+        warm_storage: "SyncWarmStorageService",
+        sp_registry,
+        fallback_retriever: Optional[PieceRetriever] = None,
+        timeout: float = 30.0,
+    ) -> None:
+        self._warm_storage = warm_storage
+        self._sp_registry = sp_registry
+        self._fallback = fallback_retriever
+        self._timeout = timeout
+        self._client = httpx.Client(timeout=timeout)
+
+    def _get_pdp_endpoint(self, provider_id: int) -> Optional[str]:
+        """Get PDP service URL for a provider."""
+        try:
+            product = self._sp_registry.get_provider_with_product(provider_id, 1)
+            for i, key in enumerate(product.product.capability_keys):
+                if key == "serviceURL" and i < len(product.product_capability_values):
+                    return product.product_capability_values[i]
+        except Exception:
+            pass
+        return None
+
+    def _find_providers(
+        self,
+        client_address: str,
+        provider_address: Optional[str] = None,
+    ) -> List["ProviderInfo"]:
+        """Find providers that can serve pieces for a client."""
+        
+        if provider_address is not None:
+            # Direct provider case
+            provider = self._sp_registry.get_provider_by_address(provider_address)
+            if provider is None:
+                raise ValueError(f"Provider {provider_address} not found in registry")
+            return [provider]
+        
+        # Get client's datasets with details
+        datasets = self._warm_storage.get_client_data_sets_with_details(client_address)
+        
+        # Filter for live datasets with pieces
+        valid_datasets = [
+            ds for ds in datasets
+            if ds.is_live and ds.active_piece_count > 0
+        ]
+        
+        if not valid_datasets:
+            raise ValueError(f"No active datasets with data found for client {client_address}")
+        
+        # Get unique provider IDs
+        unique_provider_ids = list(set(ds.provider_id for ds in valid_datasets))
+        
+        # Fetch provider info for each
+        providers = []
+        for pid in unique_provider_ids:
+            try:
+                provider = self._sp_registry.get_provider(pid)
+                if provider and provider.is_active:
+                    providers.append(provider)
+            except Exception:
+                continue
+        
+        if not providers:
+            raise ValueError("No valid providers found")
+        
+        return providers
+
+    def _try_fetch_from_provider(
+        self,
+        provider: "ProviderInfo",
+        piece_cid: str,
+    ) -> Optional[bytes]:
+        """Try to fetch a piece from a specific provider."""
+        endpoint = self._get_pdp_endpoint(provider.provider_id)
+        if not endpoint:
+            return None
+        
+        try:
+            # First check if provider has the piece
+            find_resp = self._client.get(
+                f"{endpoint.rstrip('/')}/pdp/piece",
+                params={"pieceCid": piece_cid},
+            )
+            if find_resp.status_code != 200:
+                return None
+            
+            # Download the piece
+            download_resp = self._client.get(
+                f"{endpoint.rstrip('/')}/pdp/piece/{piece_cid}",
+            )
+            if download_resp.status_code == 200:
+                return download_resp.content
+            
+        except Exception:
+            pass
+        
+        return None
+
+    def fetch_piece(
+        self,
+        piece_cid: str,
+        client_address: str,
+        provider_address: Optional[str] = None,
+        parallel: bool = True,
+    ) -> bytes:
+        """
+        Fetch a piece by CID, trying multiple providers if needed.
+        
+        Args:
+            piece_cid: The piece CID to fetch
+            client_address: The client address to look up datasets for
+            provider_address: Optional specific provider to use
+            parallel: Whether to try providers in parallel (default: True)
+            
+        Returns:
+            The piece data as bytes
+            
+        Raises:
+            ValueError: If piece cannot be found on any provider
+        """
+        try:
+            providers = self._find_providers(client_address, provider_address)
+        except ValueError as e:
+            if self._fallback:
+                return self._fallback.fetch_piece(piece_cid, client_address, provider_address)
+            raise
+        
+        if parallel and len(providers) > 1:
+            # Try all providers in parallel, return first success
+            with concurrent.futures.ThreadPoolExecutor(max_workers=len(providers)) as executor:
+                futures = {
+                    executor.submit(self._try_fetch_from_provider, p, piece_cid): p
+                    for p in providers
+                }
+                
+                for future in concurrent.futures.as_completed(futures):
+                    result = future.result()
+                    if result is not None:
+                        # Cancel remaining futures
+                        for f in futures:
+                            f.cancel()
+                        return result
+        else:
+            # Try providers sequentially
+            for provider in providers:
+                result = self._try_fetch_from_provider(provider, piece_cid)
+                if result is not None:
+                    return result
+        
+        # All providers failed
+        if self._fallback:
+            return self._fallback.fetch_piece(piece_cid, client_address, provider_address)
+        
+        raise ValueError(f"Failed to retrieve piece {piece_cid} from any provider")
+
+    def close(self) -> None:
+        """Close the HTTP client."""
+        self._client.close()
+
+    def __enter__(self) -> "ChainRetriever":
+        return self
+
+    def __exit__(self, *args) -> None:
+        self.close()

pynapse/session/__init__.py

@@ -0,0 +1,12 @@
+from .key import SessionKey
+from .permissions import ALL_PERMISSIONS, SESSION_KEY_PERMISSIONS, get_permission_from_type_hash
+from .registry import AsyncSessionKeyRegistry, SyncSessionKeyRegistry
+
+__all__ = [
+    "SessionKey",
+    "ALL_PERMISSIONS",
+    "SESSION_KEY_PERMISSIONS",
+    "get_permission_from_type_hash",
+    "AsyncSessionKeyRegistry",
+    "SyncSessionKeyRegistry",
+]

pynapse/session/key.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Dict, Iterable, List, Optional
+
+from pynapse.core.chains import Chain
+from .permissions import ALL_PERMISSIONS
+from .registry import SyncSessionKeyRegistry
+
+
+class SessionKey:
+    def __init__(self, chain: Chain, registry: SyncSessionKeyRegistry, owner_address: str, session_key_address: str) -> None:
+        self._chain = chain
+        self._registry = registry
+        self._owner_address = owner_address
+        self._session_key_address = session_key_address
+
+    def fetch_expiries(self, permissions: Iterable[str] = ALL_PERMISSIONS) -> Dict[str, int]:
+        expiries: Dict[str, int] = {}
+        for permission in permissions:
+            expiries[permission] = self._registry.authorization_expiry(
+                self._owner_address, self._session_key_address, permission
+            )
+        return expiries
+
+    def login(self, expiry: int, permissions: Iterable[str] = ALL_PERMISSIONS, origin: str = "unknown") -> str:
+        return self._registry.login(self._owner_address, self._session_key_address, expiry, permissions, origin)
+
+    def revoke(self, permissions: Iterable[str] = ALL_PERMISSIONS, origin: str = "unknown") -> str:
+        return self._registry.revoke(self._owner_address, self._session_key_address, permissions, origin)

pynapse/session/permissions.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from typing import Dict, List
+
+from eth_utils import keccak
+
+from pynapse.core.typed_data import EIP712_TYPES
+
+SessionKeyPermission = str
+
+ALL_PERMISSIONS: List[SessionKeyPermission] = [
+    "CreateDataSet",
+    "AddPieces",
+    "SchedulePieceRemovals",
+    "DeleteDataSet",
+]
+
+
+def _dependencies(types: Dict[str, List[Dict[str, str]]], primary: str) -> List[str]:
+    deps = []
+    for field in types[primary]:
+        t = field["type"].replace("[]", "")
+        if t in types and t not in deps and t != primary:
+            deps.append(t)
+            deps.extend([d for d in _dependencies(types, t) if d not in deps])
+    return deps
+
+
+def _encode_type(types: Dict[str, List[Dict[str, str]]], primary: str) -> str:
+    deps = _dependencies(types, primary)
+    deps = sorted(deps)
+    type_list = [primary] + deps
+    parts = []
+    for t in type_list:
+        fields = ",".join([f"{f['type']} {f['name']}" for f in types[t]])
+        parts.append(f"{t}({fields})")
+    return "".join(parts)
+
+
+def type_hash(primary_type: str) -> str:
+    encoded = _encode_type(EIP712_TYPES, primary_type)
+    return "0x" + keccak(text=encoded).hex()
+
+
+SESSION_KEY_PERMISSIONS: Dict[SessionKeyPermission, str] = {
+    "CreateDataSet": type_hash("CreateDataSet"),
+    "AddPieces": type_hash("AddPieces"),
+    "SchedulePieceRemovals": type_hash("SchedulePieceRemovals"),
+    "DeleteDataSet": type_hash("DeleteDataSet"),
+}
+
+
+def get_permission_from_type_hash(type_hash_value: str) -> SessionKeyPermission:
+    for perm, h in SESSION_KEY_PERMISSIONS.items():
+        if h.lower() == type_hash_value.lower():
+            return perm
+    raise ValueError(f"Permission not found for type hash: {type_hash_value}")

pynapse/session/registry.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+from typing import Dict, Iterable, List, Optional
+
+from eth_account import Account
+from web3 import AsyncWeb3, Web3
+
+from pynapse.contracts import SESSION_KEY_REGISTRY_ABI
+from pynapse.core.chains import Chain
+from .permissions import SESSION_KEY_PERMISSIONS
+
+
+class SyncSessionKeyRegistry:
+    def __init__(self, web3: Web3, chain: Chain, private_key: Optional[str] = None) -> None:
+        self._web3 = web3
+        self._chain = chain
+        self._private_key = private_key
+        self._contract = web3.eth.contract(address=chain.contracts.session_key_registry, abi=SESSION_KEY_REGISTRY_ABI)
+
+    def authorization_expiry(self, address: str, session_key_address: str, permission: str) -> int:
+        perm = SESSION_KEY_PERMISSIONS[permission]
+        return int(self._contract.functions.authorizationExpiry(address, session_key_address, perm).call())
+
+    def login(self, account: str, session_key_address: str, expires_at: int, permissions: Iterable[str], origin: str) -> str:
+        if not self._private_key:
+            raise ValueError("private_key required")
+        perm_hashes = [SESSION_KEY_PERMISSIONS[p] for p in permissions]
+        txn = self._contract.functions.login(session_key_address, expires_at, perm_hashes, origin).build_transaction(
+            {
+                "from": account,
+                "nonce": self._web3.eth.get_transaction_count(account),
+            }
+        )
+        signed = self._web3.eth.account.sign_transaction(txn, private_key=self._private_key)
+        tx_hash = self._web3.eth.send_raw_transaction(signed.rawTransaction)
+        return tx_hash.hex()
+
+    def revoke(self, account: str, session_key_address: str, permissions: Iterable[str], origin: str) -> str:
+        if not self._private_key:
+            raise ValueError("private_key required")
+        perm_hashes = [SESSION_KEY_PERMISSIONS[p] for p in permissions]
+        txn = self._contract.functions.revoke(session_key_address, perm_hashes, origin).build_transaction(
+            {
+                "from": account,
+                "nonce": self._web3.eth.get_transaction_count(account),
+            }
+        )
+        signed = self._web3.eth.account.sign_transaction(txn, private_key=self._private_key)
+        tx_hash = self._web3.eth.send_raw_transaction(signed.rawTransaction)
+        return tx_hash.hex()
+
+
+class AsyncSessionKeyRegistry:
+    def __init__(self, web3: AsyncWeb3, chain: Chain, private_key: Optional[str] = None) -> None:
+        self._web3 = web3
+        self._chain = chain
+        self._private_key = private_key
+        self._contract = web3.eth.contract(address=chain.contracts.session_key_registry, abi=SESSION_KEY_REGISTRY_ABI)
+
+    async def authorization_expiry(self, address: str, session_key_address: str, permission: str) -> int:
+        perm = SESSION_KEY_PERMISSIONS[permission]
+        return int(await self._contract.functions.authorizationExpiry(address, session_key_address, perm).call())
+
+    async def login(self, account: str, session_key_address: str, expires_at: int, permissions: Iterable[str], origin: str) -> str:
+        if not self._private_key:
+            raise ValueError("private_key required")
+        perm_hashes = [SESSION_KEY_PERMISSIONS[p] for p in permissions]
+        txn = await self._contract.functions.login(session_key_address, expires_at, perm_hashes, origin).build_transaction(
+            {
+                "from": account,
+                "nonce": await self._web3.eth.get_transaction_count(account),
+            }
+        )
+        signed = Account.sign_transaction(txn, private_key=self._private_key)
+        tx_hash = await self._web3.eth.send_raw_transaction(signed.rawTransaction)
+        return tx_hash.hex()
+
+    async def revoke(self, account: str, session_key_address: str, permissions: Iterable[str], origin: str) -> str:
+        if not self._private_key:
+            raise ValueError("private_key required")
+        perm_hashes = [SESSION_KEY_PERMISSIONS[p] for p in permissions]
+        txn = await self._contract.functions.revoke(session_key_address, perm_hashes, origin).build_transaction(
+            {
+                "from": account,
+                "nonce": await self._web3.eth.get_transaction_count(account),
+            }
+        )
+        signed = Account.sign_transaction(txn, private_key=self._private_key)
+        tx_hash = await self._web3.eth.send_raw_transaction(signed.rawTransaction)
+        return tx_hash.hex()

pynapse/sp_registry/__init__.py

@@ -0,0 +1,11 @@
+from .service import AsyncSPRegistryService, SyncSPRegistryService
+from .types import PDPOffering, ProviderInfo, ProviderRegistrationInfo, ProviderWithProduct
+
+__all__ = [
+    "AsyncSPRegistryService",
+    "SyncSPRegistryService",
+    "ProviderInfo",
+    "ProviderWithProduct",
+    "ProviderRegistrationInfo",
+    "PDPOffering",
+]

pynapse/sp_registry/capabilities.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from typing import Dict, List
+
+from web3 import Web3
+
+
+def capabilities_list_to_object(keys: List[str], values: List[bytes]) -> Dict[str, str]:
+    capabilities: Dict[str, str] = {}
+    for key, value in zip(keys, values):
+        capabilities[key] = Web3.to_hex(value)
+    return capabilities
+
+
+def decode_address_capability(capability_value: bytes | str) -> str:
+    if isinstance(capability_value, str):
+        hex_value = capability_value
+    else:
+        hex_value = Web3.to_hex(capability_value)
+
+    if len(hex_value) > 42:
+        return "0x" + hex_value[-40:]
+    if len(hex_value) < 42:
+        return Web3.to_checksum_address(hex_value.rjust(42, "0"))
+    return Web3.to_checksum_address(hex_value)