hippius 0.1.13__py3-none-any.whl → 0.2.0__py3-none-any.whl

hippius_sdk/ipfs_core.py

@@ -0,0 +1,216 @@
+import json
+import os
+from typing import Any, Dict
+
+import httpx
+
+
+class AsyncIPFSClient:
+    """
+    Asynchronous IPFS client using httpx.
+    """
+
+    def __init__(
+        self, api_url: str = "http://localhost:5001", gateway: str = "https://ipfs.io"
+    ):
+        # Handle multiaddr format
+        if api_url and api_url.startswith("/"):
+            # Extract host and port from multiaddr
+            try:
+                parts = api_url.split("/")
+                # Handle /ip4/127.0.0.1/tcp/5001
+                if len(parts) >= 5 and parts[1] in ["ip4", "ip6"]:
+                    host = parts[2]
+                    port = parts[4]
+                    api_url = f"https://{host}:{port}"
+                    print(f"Converted multiaddr {api_url} to HTTP URL {api_url}")
+                else:
+                    print(f"Warning: Unsupported multiaddr format: {api_url}")
+                    print("Falling back to default: http://localhost:5001")
+                    api_url = "http://localhost:5001"
+            except Exception as e:
+                print(f"Error parsing multiaddr: {e}")
+                print("Falling back to default: http://localhost:5001")
+                api_url = "http://localhost:5001"
+        self.api_url = api_url
+        self.gateway = gateway
+        self.client = httpx.AsyncClient(timeout=60.0)
+
+    async def close(self):
+        """Close the httpx client."""
+        await self.client.aclose()
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.close()
+
+    async def add_file(self, file_path: str) -> Dict[str, Any]:
+        """
+        Add a file to IPFS.
+
+        Args:
+            file_path: Path to the file to add
+
+        Returns:
+            Dict containing the CID and other information
+        """
+        with open(file_path, "rb") as f:
+            files = {"file": f}
+            response = await self.client.post(f"{self.api_url}/api/v0/add", files=files)
+            response.raise_for_status()
+            return response.json()
+
+    async def add_bytes(self, data: bytes, filename: str = "file") -> Dict[str, Any]:
+        """
+        Add bytes to IPFS.
+
+        Args:
+            data: Bytes to add
+            filename: Name to give the file (default: "file")
+
+        Returns:
+            Dict containing the CID and other information
+        """
+        files = {"file": (filename, data)}
+        response = await self.client.post(f"{self.api_url}/api/v0/add", files=files)
+        response.raise_for_status()
+        return response.json()
+
+    async def add_str(self, content: str, filename: str = "file") -> Dict[str, Any]:
+        """
+        Add a string to IPFS.
+
+        Args:
+            content: String to add
+            filename: Name to give the file (default: "file")
+
+        Returns:
+            Dict containing the CID and other information
+        """
+        return await self.add_bytes(content.encode(), filename)
+
+    async def cat(self, cid: str) -> bytes:
+        """
+        Retrieve content from IPFS by its CID.
+
+        Args:
+            cid: Content Identifier to retrieve
+
+        Returns:
+            Content as bytes
+        """
+        response = await self.client.post(f"{self.api_url}/api/v0/cat?arg={cid}")
+        response.raise_for_status()
+        return response.content
+
+    async def pin(self, cid: str) -> Dict[str, Any]:
+        """
+        Pin content by CID.
+
+        Args:
+            cid: Content Identifier to pin
+
+        Returns:
+            Response from the IPFS node
+        """
+        response = await self.client.post(f"{self.api_url}/api/v0/pin/add?arg={cid}")
+        response.raise_for_status()
+        return response.json()
+
+    async def ls(self, cid: str) -> Dict[str, Any]:
+        """
+        List objects linked to the specified CID.
+
+        Args:
+            cid: Content Identifier
+
+        Returns:
+            Dict with links information
+        """
+        response = await self.client.post(f"{self.api_url}/api/v0/ls?arg={cid}")
+        response.raise_for_status()
+        return response.json()
+
+    async def exists(self, cid: str) -> bool:
+        """
+        Check if content exists.
+
+        Args:
+            cid: Content Identifier to check
+
+        Returns:
+            True if content exists, False otherwise
+        """
+        try:
+            await self.client.head(f"{self.gateway}/ipfs/{cid}")
+            return True
+        except httpx.HTTPError:
+            return False
+
+    async def download_file(self, cid: str, output_path: str) -> str:
+        """
+        Download content from IPFS to a file.
+
+        Args:
+            cid: Content identifier
+            output_path: Path where to save the file
+
+        Returns:
+            Path to the saved file
+        """
+        content = await self.cat(cid)
+        with open(output_path, "wb") as f:
+            f.write(content)
+        return output_path
+
+    async def add_directory(
+        self, dir_path: str, recursive: bool = True
+    ) -> Dict[str, Any]:
+        """
+        Add a directory to IPFS.
+
+        Args:
+            dir_path: Path to the directory to add
+
+        Returns:
+            Dict containing the CID and other information about the directory
+
+        Raises:
+            FileNotFoundError: If the directory doesn't exist
+            httpx.HTTPError: If the IPFS API request fails
+        """
+        if not os.path.isdir(dir_path):
+            raise FileNotFoundError(f"Directory {dir_path} not found")
+
+        # Collect all files in the directory
+        files = []
+        for root, _, filenames in os.walk(dir_path):
+            for filename in filenames:
+                file_path = os.path.join(root, filename)
+                rel_path = os.path.relpath(file_path, dir_path)
+
+                with open(file_path, "rb") as f:
+                    file_content = f.read()
+
+                # Add the file to the multipart request
+                files.append(
+                    ("file", (rel_path, file_content, "application/octet-stream"))
+                )
+
+        # Make the request with directory flags
+        response = await self.client.post(
+            f"{self.api_url}/api/v0/add?recursive=true&wrap-with-directory=true",
+            files=files,
+            timeout=300.0,  # 5 minute timeout for directory uploads
+        )
+        response.raise_for_status()
+
+        # The IPFS API returns a JSON object for each file, one per line
+        # The last one should be the directory itself
+        lines = response.text.strip().split("\n")
+        if not lines:
+            raise ValueError("Empty response from IPFS API")
+
+        return json.loads(lines[-1])

hippius_sdk/substrate.py

@@ -6,8 +6,9 @@ Note: This functionality is coming soon and not implemented yet.
 
 import json
 import os
+import tempfile
 import uuid
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Type, Union
 
 from dotenv import load_dotenv
 from substrateinterface import Keypair, SubstrateInterface
@@ -19,6 +20,7 @@ from hippius_sdk.config import (
     get_seed_phrase,
     set_seed_phrase,
 )
+from hippius_sdk.utils import hex_to_ipfs_cid
 
 # Load environment variables
 load_dotenv()
@@ -202,7 +204,7 @@ class SubstrateClient:
             print(f"Warning: Could not create keypair from seed phrase: {e}")
             print(f"Keypair will be created when needed")
 
-    def storage_request(
+    async def storage_request(
         self, files: List[Union[FileInput, Dict[str, str]]], miner_ids: List[str] = None
     ) -> str:
         """
@@ -276,8 +278,7 @@ class SubstrateClient:
             print(f"Created file list with {len(file_list)} entries")
 
             # Step 2: Upload the JSON file to IPFS
-            import tempfile
-
+            # Defer import to avoid circular imports
             from hippius_sdk.ipfs import IPFSClient
 
             ipfs_client = IPFSClient()
@@ -291,7 +292,7 @@ class SubstrateClient:
 
             try:
                 print("Uploading file list to IPFS...")
-                upload_result = ipfs_client.upload_file(temp_file_path)
+                upload_result = await ipfs_client.upload_file(temp_file_path)
                 files_list_cid = upload_result["cid"]
                 print(f"File list uploaded to IPFS with CID: {files_list_cid}")
             finally:
@@ -313,19 +314,42 @@ class SubstrateClient:
 
             # Create the call to the marketplace
             print(f"Call parameters: {json.dumps(call_params, indent=2)}")
-            call = self._substrate.compose_call(
-                call_module="Marketplace",
-                call_function="storage_request",
-                call_params=call_params,
-            )
+            try:
+                call = self._substrate.compose_call(
+                    call_module="Marketplace",
+                    call_function="storage_request",
+                    call_params=call_params,
+                )
+            except Exception as e:
+                print(f"Warning: Error composing call: {e}")
+                print("Attempting to use IpfsPallet.storeFile instead...")
+
+                # Try with IpfsPallet.storeFile as an alternative
+                alt_call_params = {
+                    "fileHash": files_list_cid,
+                    "fileName": f"files_list_{uuid.uuid4()}",  # Generate a unique ID
+                }
+                call = self._substrate.compose_call(
+                    call_module="IpfsPallet",
+                    call_function="storeFile",
+                    call_params=alt_call_params,
+                )
 
             # Get payment info to estimate the fee
             payment_info = self._substrate.get_payment_info(
                 call=call, keypair=self._keypair
             )
 
+            print(f"Payment info: {json.dumps(payment_info, indent=2)}")
+
+            # Convert partialFee from Substrate (10^18 units) to a more readable format
             estimated_fee = payment_info.get("partialFee", 0)
-            print(f"Estimated transaction fee: {estimated_fee}")
+            estimated_fee_formatted = (
+                float(estimated_fee) / 1_000_000_000_000_000_000 if estimated_fee else 0
+            )
+            print(
+                f"Estimated transaction fee: {estimated_fee} ({estimated_fee_formatted:.10f} tokens)"
+            )
 
             # Create a signed extrinsic
             extrinsic = self._substrate.create_signed_extrinsic(
@@ -361,7 +385,7 @@ class SubstrateClient:
 
         return "simulated-tx-hash"
 
-    def store_cid(
+    async def store_cid(
         self, cid: str, filename: str = None, metadata: Optional[Dict[str, Any]] = None
     ) -> str:
         """
@@ -376,7 +400,7 @@ class SubstrateClient:
             str: Transaction hash
         """
         file_input = FileInput(file_hash=cid, file_name=filename or "unnamed_file")
-        return self.storage_request([file_input])
+        return await self.storage_request([file_input])
 
     def get_cid_metadata(self, cid: str) -> Dict[str, Any]:
         """
@@ -569,7 +593,7 @@ class SubstrateClient:
             print(error_msg)
             raise ValueError(error_msg)
 
-    def get_user_files(
+    async def get_user_files(
         self,
         account_address: Optional[str] = None,
         truncate_miners: bool = True,
@@ -604,9 +628,9 @@ class SubstrateClient:
         """
         # For backward compatibility, this method now calls get_user_files_from_profile
         # with appropriate conversions
-        return self.get_user_files_from_profile(account_address)
+        return await self.get_user_files_from_profile(account_address)
 
-    def get_user_files_from_profile(
+    async def get_user_files_from_profile(
         self,
         account_address: Optional[str] = None,
     ) -> List[Dict[str, Any]]:
@@ -680,12 +704,13 @@ class SubstrateClient:
             print(f"Decoded IPFS CID: {profile_cid}")
 
             # Fetch the profile JSON from IPFS
+            # Defer import to avoid circular imports
             from hippius_sdk.ipfs import IPFSClient
 
             ipfs_client = IPFSClient()
 
             print(f"Fetching user profile from IPFS: {profile_cid}")
-            profile_data = ipfs_client.cat(profile_cid)
+            profile_data = await ipfs_client.cat(profile_cid)
 
             # Parse the JSON content
             if not profile_data.get("is_text", False):
@@ -761,6 +786,7 @@ class SubstrateClient:
                     file.get("size")
                     or file.get("fileSize")
                     or file.get("file_size")
+                    or file.get("file_size_in_bytes")
                     or 0
                 )
 
@@ -772,7 +798,7 @@ class SubstrateClient:
                         "miner_ids", []
                     ),  # Try to get miners if available
                     "miner_count": len(file.get("miner_ids", [])),  # Count the miners
-                    "file_size": file_size,
+                    "file_size": file_size or 0,
                 }
 
                 # Add formatted file size if available
@@ -796,56 +822,205 @@ class SubstrateClient:
             print(error_msg)
             raise ValueError(error_msg)
 
-    def _hex_to_ipfs_cid(self, hex_string: str) -> str:
+    def get_pinning_status(
+        self, account_address: Optional[str] = None
+    ) -> List[Dict[str, Any]]:
         """
-        Convert a hex-encoded IPFS CID to a regular IPFS CID.
+        Get the status of file pinning requests for an account.
+
+        This method queries the blockchain for all storage requests made by the user
+        to check their pinning status.
 
         Args:
-            hex_string: Hex string representation of an IPFS CID
+            account_address: Substrate account address (uses keypair address if not specified)
+                             Format: 5HoreGVb17XhY3wanDvzoAWS7yHYbc5uMteXqRNTiZ6Txkqq
 
         Returns:
-            str: Regular IPFS CID
+            List[Dict[str, Any]]: List of storage requests with their status information:
+                {
+                    "cid": str,              # The IPFS CID of the file
+                    "file_name": str,        # The name of the file
+                    "total_replicas": int,   # Total number of replicas requested
+                    "owner": str,            # Owner's address
+                    "created_at": int,       # Block number when request was created
+                    "last_charged_at": int,  # Block number when last charged
+                    "miner_ids": List[str],  # List of miners assigned to pin the file
+                    "selected_validator": str, # Selected validator address
+                    "is_assigned": bool,     # Whether request has been assigned to miners
+                }
+
+        Raises:
+            ConnectionError: If connection to Substrate fails
+            ValueError: If query fails or no requests found
         """
-        # First, try to decode as ASCII if it's a hex representation of ASCII characters
         try:
-            if hex_string.startswith("0x"):
-                hex_string = hex_string[2:]
+            # Initialize Substrate connection if not already connected
+            if not hasattr(self, "_substrate") or self._substrate is None:
+                print("Initializing Substrate connection...")
+                self._substrate = SubstrateInterface(
+                    url=self.url,
+                    ss58_format=42,  # Substrate default
+                    type_registry_preset="substrate-node-template",
+                )
+                print(f"Connected to Substrate node at {self.url}")
 
-            bytes_data = bytes.fromhex(hex_string)
-            ascii_str = bytes_data.decode("ascii")
+            # Use provided account address or default to keypair/configured address
+            if not account_address:
+                if self._account_address:
+                    account_address = self._account_address
+                    print(f"Using account address: {account_address}")
+                else:
+                    # Try to get the address from the keypair (requires seed phrase)
+                    if not self._ensure_keypair():
+                        raise ValueError("No account address available")
+                    account_address = self._keypair.ss58_address
+                    print(f"Using keypair address: {account_address}")
 
-            # If the decoded string starts with a valid CID prefix, return it
-            if ascii_str.startswith(("Qm", "bafy", "bafk", "bafyb", "bafzb", "b")):
-                return ascii_str
-        except Exception:
-            # If ASCII decoding fails, continue with other methods
-            pass
+            # Query the blockchain for storage requests
+            print(f"Querying storage requests for account: {account_address}")
+            try:
+                # First, try with query_map which is more suitable for iterating over collections
+                result = self._substrate.query_map(
+                    module="IpfsPallet",
+                    storage_function="UserStorageRequests",
+                    params=[account_address],
+                )
+                results_list = list(result)
+            except Exception as e:
+                print(f"Error with query_map: {e}")
+                try:
+                    # Try again with query to double check storage function requirements
+                    result = self._substrate.query(
+                        module="IpfsPallet",
+                        storage_function="UserStorageRequests",
+                        params=[
+                            account_address,
+                            None,
+                        ],  # Try with a None second parameter
+                    )
+
+                    # If the query returns a nested structure, extract it
+                    if result.value and isinstance(result.value, list):
+                        # Convert to a list format similar to query_map for processing
+                        results_list = []
+                        for item in result.value:
+                            if isinstance(item, list) and len(item) >= 2:
+                                key = item[0]
+                                value = item[1]
+                                results_list.append((key, value))
+                    else:
+                        # If it's not a nested structure, use a simpler format
+                        results_list = [(None, result.value)] if result.value else []
+                except Exception as e_inner:
+                    print(f"Error with fallback query: {e_inner}")
+                    # If both methods fail, return an empty list
+                    results_list = []
+
+            # Process the storage requests
+            storage_requests = []
+
+            if not results_list:
+                print(f"No storage requests found for account: {account_address}")
+                return []
+
+            print(f"Found {len(results_list)} storage request entries")
+
+            for i, (key, value) in enumerate(results_list):
+                try:
+                    # For debugging, print raw data
+                    print(f"Entry {i+1}:")
+                    print(f"  Raw key: {key}, type: {type(key)}")
+                    print(f"  Raw value: {value}, type: {type(value)}")
+
+                    # Extract file hash from key if possible
+                    file_hash_hex = None
+                    if key is not None:
+                        if hasattr(key, "hex"):
+                            file_hash_hex = key.hex()
+                        elif isinstance(key, bytes):
+                            file_hash_hex = key.hex()
+                        elif isinstance(key, str) and key.startswith("0x"):
+                            file_hash_hex = key[2:]
+                        else:
+                            file_hash_hex = str(key)
+
+                    # Try to extract value data
+                    request_data = None
+                    if isinstance(value, dict):
+                        request_data = value
+                    elif hasattr(value, "get"):
+                        request_data = value
+                    elif hasattr(value, "__dict__"):
+                        # Convert object to dict
+                        request_data = {
+                            k: getattr(value, k)
+                            for k in dir(value)
+                            if not k.startswith("_") and not callable(getattr(value, k))
+                        }
+
+                    # If we can't extract data, just use value as string for debugging
+                    if request_data is None:
+                        request_data = {"raw_value": str(value)}
+
+                    # Create formatted request with available data
+                    formatted_request = {"raw_key": str(key), "raw_value": str(value)}
+
+                    # Directly extract file_name from the value if it's a dict-like object
+                    if hasattr(value, "get"):
+                        if value.get("file_name"):
+                            formatted_request["file_name"] = value.get("file_name")
+                        elif value.get("fileName"):
+                            formatted_request["file_name"] = value.get("fileName")
+
+                    # Add CID if we have it
+                    if file_hash_hex:
+                        file_cid = self._hex_to_ipfs_cid(file_hash_hex)
+                        formatted_request["cid"] = file_cid
+
+                    # Add other fields from request_data if available
+                    for source_field, target_field in [
+                        ("fileName", "file_name"),
+                        ("totalReplicas", "total_replicas"),
+                        ("owner", "owner"),
+                        ("createdAt", "created_at"),
+                        ("lastChargedAt", "last_charged_at"),
+                        ("minerIds", "miner_ids"),
+                        ("selectedValidator", "selected_validator"),
+                        ("isAssigned", "is_assigned"),
+                        # Add variants that might appear differently in the chain storage
+                        ("file_name", "file_name"),
+                        ("file_hash", "file_hash"),
+                        ("total_replicas", "total_replicas"),
+                    ]:
+                        if source_field in request_data:
+                            formatted_request[target_field] = request_data[source_field]
+                        # Fallback to attribute access for different types of objects
+                        elif hasattr(value, source_field):
+                            formatted_request[target_field] = getattr(
+                                value, source_field
+                            )
+
+                    storage_requests.append(formatted_request)
+
+                except Exception as e:
+                    print(f"Error processing request entry {i+1}: {e}")
+
+            print(f"Successfully processed {len(storage_requests)} storage requests")
+            return storage_requests
 
-        # Try to decode as a binary CID
-        try:
-            import base58
-
-            if hex_string.startswith("0x"):
-                hex_string = hex_string[2:]
-
-            binary_data = bytes.fromhex(hex_string)
-
-            # Check if it matches CIDv0 pattern (starts with 0x12, 0x20)
-            if (
-                len(binary_data) > 2
-                and binary_data[0] == 0x12
-                and binary_data[1] == 0x20
-            ):
-                # CIDv0 (Qm...)
-                return base58.b58encode(binary_data).decode("utf-8")
-
-            # If it doesn't match CIDv0, for CIDv1 just return the hex without 0x prefix
-            # since adding 0x breaks IPFS gateway URLs
-            return hex_string
-        except ImportError:
-            # If base58 is not available
-            print("Warning: base58 module not available for proper CID conversion")
-            return hex_string
         except Exception as e:
-            print(f"Error converting hex to CID: {e}")
-            return hex_string
+            error_msg = f"Error querying storage requests: {str(e)}"
+            print(error_msg)
+            raise ValueError(error_msg)
+
+    def _hex_to_ipfs_cid(self, hex_string: str) -> str:
+        """
+        Convert a hex-encoded IPFS CID to a regular IPFS CID.
+
+        Args:
+            hex_string: Hex string representation of an IPFS CID
+
+        Returns:
+            str: Regular IPFS CID
+        """
+        return hex_to_ipfs_cid(hex_string)