hippius 0.2.4__py3-none-any.whl → 0.2.6__py3-none-any.whl

hippius_sdk/ipfs.py

@@ -16,6 +16,13 @@ import httpx
 import requests
 
 from hippius_sdk.config import get_config_value, get_encryption_key
+from hippius_sdk.errors import (
+    HippiusAlreadyDeletedError,
+    HippiusFailedIPFSUnpin,
+    HippiusFailedSubstrateDelete,
+    HippiusIPFSConnectionError,
+    HippiusMetadataError,
+)
 from hippius_sdk.ipfs_core import AsyncIPFSClient
 from hippius_sdk.substrate import FileInput, SubstrateClient
 from hippius_sdk.utils import format_cid, format_size
@@ -436,15 +443,18 @@ class IPFSClient:
         output_path: str,
         decrypt: Optional[bool] = None,
         max_retries: int = 3,
+        skip_directory_check: bool = False,
     ) -> Dict[str, Any]:
         """
         Download a file from IPFS with optional decryption.
+        Supports downloading directories - in that case, a directory structure will be created.
 
         Args:
             cid: Content Identifier (CID) of the file to download
-            output_path: Path where the downloaded file will be saved
+            output_path: Path where the downloaded file/directory will be saved
             decrypt: Whether to decrypt the file (overrides default)
             max_retries: Maximum number of retry attempts (default: 3)
+            skip_directory_check: If True, skips directory check (treats as file)
 
         Returns:
             Dict[str, Any]: Dictionary containing download results:
@@ -454,6 +464,7 @@ class IPFSClient:
                 - size_formatted: Human-readable file size
                 - elapsed_seconds: Time taken for the download in seconds
                 - decrypted: Whether the file was decrypted
+                - is_directory: Whether the download was a directory
 
         Raises:
             requests.RequestException: If the download fails
@@ -461,6 +472,47 @@ class IPFSClient:
         """
         start_time = time.time()
 
+        # Skip directory check if requested (important for erasure code chunks)
+        is_directory = False
+        if not skip_directory_check:
+            # Use the improved ls function to properly detect directories
+            try:
+                # The ls function now properly detects directories
+                ls_result = await self.client.ls(cid)
+                is_directory = ls_result.get("is_directory", False)
+            except Exception:
+                # If ls fails, we'll proceed as if it's a file
+                pass
+
+        # If it's a directory, handle it differently
+        if is_directory:
+            # For directories, we don't need to decrypt each file during the initial download
+            # We'll use the AsyncIPFSClient's download_directory method directly
+            try:
+                await self.client.download_directory(cid, output_path)
+
+                # Calculate the total size of the directory
+                total_size = 0
+                for root, _, files in os.walk(output_path):
+                    for file in files:
+                        file_path = os.path.join(root, file)
+                        total_size += os.path.getsize(file_path)
+
+                elapsed_time = time.time() - start_time
+
+                return {
+                    "success": True,
+                    "output_path": output_path,
+                    "size_bytes": total_size,
+                    "size_formatted": self.format_size(total_size),
+                    "elapsed_seconds": round(elapsed_time, 2),
+                    "decrypted": False,  # Directories aren't decrypted as a whole
+                    "is_directory": True,
+                }
+            except Exception as e:
+                raise RuntimeError(f"Failed to download directory: {str(e)}")
+
+        # For regular files, use the existing logic
         # Determine if we should decrypt
         should_decrypt = self.encrypt_by_default if decrypt is None else decrypt
 
@@ -482,38 +534,14 @@ class IPFSClient:
             else:
                 download_path = output_path
 
-            # Download the file with retry logic
-            retries = 0
-            while retries < max_retries:
-                try:
-                    # Download the file
-                    url = f"{self.gateway}/ipfs/{cid}"
-                    response = requests.get(url, stream=True)
-                    response.raise_for_status()
-
-                    os.makedirs(
-                        os.path.dirname(os.path.abspath(download_path)), exist_ok=True
-                    )
-
-                    with open(download_path, "wb") as f:
-                        for chunk in response.iter_content(chunk_size=8192):
-                            f.write(chunk)
-
-                    # If we reach here, download was successful
-                    break
-
-                except (requests.exceptions.RequestException, IOError) as e:
-                    # Save the error and retry
-                    retries += 1
+            # Pass the skip_directory_check parameter to the core client
+            await self.client.download_file(
+                cid, download_path, skip_directory_check=skip_directory_check
+            )
+            download_success = True
 
-                    if retries < max_retries:
-                        wait_time = 2**retries  # Exponential backoff: 2, 4, 8 seconds
-                        print(f"Download attempt {retries} failed: {str(e)}")
-                        print(f"Retrying in {wait_time} seconds...")
-                        time.sleep(wait_time)
-                    else:
-                        # Raise the last error if we've exhausted all retries
-                        raise
+            if not download_success:
+                raise RuntimeError("Failed to download file after multiple attempts")
 
             # Decrypt if needed
             if should_decrypt:
@@ -548,6 +576,7 @@ class IPFSClient:
                 "size_formatted": self.format_size(file_size_bytes),
                 "elapsed_seconds": round(elapsed_time, 2),
                 "decrypted": should_decrypt,
+                "is_directory": False,
             }
 
         finally:
@@ -1031,11 +1060,20 @@ class IPFSClient:
 
             # Step 5: Create and upload the metadata file
             metadata_path = os.path.join(temp_dir, f"{file_id}_metadata.json")
-            with open(metadata_path, "w") as f:
-                json.dump(metadata, f, indent=2)
+
+            # Use binary mode to avoid any platform-specific text encoding issues
+            with open(metadata_path, "wb") as f:
+                # Encode the JSON with UTF-8 encoding explicitly
+                metadata_json = json.dumps(metadata, indent=2, ensure_ascii=False)
+                f.write(metadata_json.encode("utf-8"))
+
+            # Verify file was written correctly
+            if os.path.getsize(metadata_path) == 0:
+                raise ValueError("Failed to write metadata file (file size is 0)")
 
             if verbose:
                 print("Uploading metadata file...")
+                print(f"Metadata file size: {os.path.getsize(metadata_path)} bytes")
 
             # Upload the metadata file to IPFS
             metadata_cid_result = await self.upload_file(
@@ -1108,9 +1146,12 @@ class IPFSClient:
             if verbose:
                 metadata_download_time = time.time() - start_time
                 print(f"Metadata downloaded in {metadata_download_time:.2f} seconds")
+                print(f"Metadata file size: {os.path.getsize(metadata_path)} bytes")
 
-            with open(metadata_path, "r") as f:
-                metadata = json.load(f)
+            # Read using binary mode to avoid any encoding issues
+            with open(metadata_path, "rb") as f:
+                metadata_content = f.read().decode("utf-8")
+                metadata = json.loads(metadata_content)
 
             # Step 2: Extract key information
             original_file = metadata["original_file"]
@@ -1191,8 +1232,12 @@ class IPFSClient:
                         async def download_chunk(cid, path, chunk_info):
                             async with encoded_chunks_semaphore:
                                 try:
+                                    # Always skip directory check for erasure code chunks
                                     await self.download_file(
-                                        cid, path, max_retries=max_retries
+                                        cid,
+                                        path,
+                                        max_retries=max_retries,
+                                        skip_directory_check=True,
                                     )
 
                                     # Read chunk data
@@ -1522,10 +1567,11 @@ class IPFSClient:
         self, cid: str, cancel_from_blockchain: bool = True
     ) -> Dict[str, Any]:
         """
-        Delete a file from IPFS and optionally cancel its storage on the blockchain.
+        Delete a file or directory from IPFS and optionally cancel its storage on the blockchain.
+        If deleting a directory, all files within the directory will be unpinned recursively.
 
         Args:
-            cid: Content Identifier (CID) of the file to delete
+            cid: Content Identifier (CID) of the file/directory to delete
             cancel_from_blockchain: Whether to also cancel the storage request from the blockchain
 
         Returns:
@@ -1540,77 +1586,117 @@ class IPFSClient:
                 "end_time": None,
                 "duration_seconds": None,
             },
+            "is_directory": False,
+            "child_files": [],
         }
 
-        # First, unpin from IPFS
+        # First check if this is a directory
         try:
-            print(f"Unpinning file from IPFS: {cid}")
-            try:
-                # Try to check if file exists in IPFS before unpinning
-                await self.exists(cid)
-            except Exception as exists_e:
-                print(f"ERROR: Error checking file existence: {exists_e}")
+            ls_result = await self.client.ls(cid)
+            is_directory = ls_result.get("is_directory", False)
+            result["is_directory"] = is_directory
+
+            # If it's a directory, recursively unpin all contained files first
+            if is_directory:
+                print(f"Detected directory: {cid}")
+                links = []
+
+                # Extract all links from the directory listing
+                if "Objects" in ls_result and len(ls_result["Objects"]) > 0:
+                    for obj in ls_result["Objects"]:
+                        if "Links" in obj:
+                            links.extend(obj["Links"])
+
+                child_files = []
+                # Unpin each item in the directory
+                for link in links:
+                    link_hash = link.get("Hash")
+                    link_name = link.get("Name", "unknown")
+                    if link_hash:
+                        child_files.append({"cid": link_hash, "name": link_name})
+                        try:
+                            # Recursively delete if it's a subdirectory
+                            link_type = link.get("Type")
+                            if (
+                                link_type == 1
+                                or str(link_type) == "1"
+                                or link_type == "dir"
+                            ):
+                                # Recursive delete, but don't cancel from blockchain (we'll do that for parent)
+                                await self.delete_file(
+                                    link_hash, cancel_from_blockchain=False
+                                )
+                            else:
+                                # Regular file unpin
+                                try:
+                                    await self.client.unpin(link_hash)
+                                    print(
+                                        f"Unpinned file: {link_name} (CID: {link_hash})"
+                                    )
+                                except Exception as unpin_error:
+                                    # Just note the error but don't let it stop the whole process
+                                    # This is common with IPFS servers that may return 500 errors for
+                                    # unpinning content that was never explicitly pinned
+                                    print(
+                                        f"Note: Could not unpin {link_name}: {str(unpin_error).split('For more information')[0]}"
+                                    )
+                        except Exception as e:
+                            print(
+                                f"Warning: Problem processing child item {link_name}: {str(e).split('For more information')[0]}"
+                            )
 
+                # Record the child files that were processed
+                result["child_files"] = child_files
+        except Exception as e:
+            print(f"Warning: Failed to check if CID is a directory: {e}")
+            # Continue with regular file unpin
+
+        # Now unpin the main file/directory
+        try:
+            print(f"Unpinning from IPFS: {cid}")
             unpin_result = await self.client.unpin(cid)
             result["unpin_result"] = unpin_result
+            result["success"] = True
             print("Successfully unpinned from IPFS")
         except Exception as e:
-            print(f"Warning: Failed to unpin file from IPFS: {e}")
-            raise
+            # Handle 500 errors from IPFS server gracefully - they often occur
+            # when the content wasn't explicitly pinned or was already unpinned
+            error_str = str(e)
+            if "500 Internal Server Error" in error_str:
+                print(
+                    f"Note: IPFS server reported content may already be unpinned: {cid}"
+                )
+                result["unpin_result"] = {"Pins": [cid]}  # Simulate successful unpin
+                result["success"] = True
+            else:
+                print(
+                    f"Warning: Failed to unpin from IPFS: {error_str.split('For more information')[0]}"
+                )
+                result["success"] = False
 
         # Then, if requested, cancel from blockchain
         if cancel_from_blockchain:
             try:
-                # Create a substrate client
-                print(f"DEBUG: Creating SubstrateClient for blockchain cancellation...")
                 substrate_client = SubstrateClient()
-                print(
-                    f"DEBUG: Substrate client created with URL: {substrate_client.url}"
-                )
-                print(f"DEBUG: Calling cancel_storage_request with CID: {cid}")
-
-                tx_hash = await substrate_client.cancel_storage_request(cid)
-                print(f"DEBUG: Received transaction hash: {tx_hash}")
-
-                # Check the return value - special cases for when blockchain cancellation isn't available
-                if tx_hash == "no-blockchain-cancellation-available":
-                    print(
-                        "Blockchain cancellation not available, but IPFS unpinning was successful"
-                    )
-                    result["blockchain_result"] = {
-                        "status": "not_available",
-                        "message": "Blockchain cancellation not available, but IPFS unpinning was successful",
-                    }
-                elif tx_hash.startswith("ipfs-unpinned-only"):
-                    error_msg = tx_hash.replace("ipfs-unpinned-only-", "")
+                await substrate_client.cancel_storage_request(cid)
+                print("Successfully cancelled storage from blockchain")
+                result["blockchain_result"] = {"success": True}
+            except Exception as e:
+                # Handle the case where the CID is not in storage requests
+                error_str = str(e)
+                if "not found in storage requests" in error_str:
                     print(
-                        f"IPFS unpinning successful, but blockchain cancellation failed: {error_msg}"
+                        "Note: Content was not found in blockchain storage requests (may already be deleted)"
                     )
                     result["blockchain_result"] = {
-                        "status": "failed",
-                        "error": error_msg,
-                        "message": "IPFS unpinning successful, but blockchain cancellation failed",
+                        "success": True,
+                        "already_deleted": True,
                     }
                 else:
-                    # Standard successful transaction
-                    result["blockchain_result"] = {
-                        "transaction_hash": tx_hash,
-                        "status": "success",
-                    }
-                    print(f"Successfully canceled storage request from blockchain")
-                    print(
-                        f"DEBUG: Blockchain cancellation succeeded with transaction hash: {tx_hash}"
-                    )
-            except Exception as e:
-                print(f"Warning: Failed to cancel storage from blockchain: {e}")
-                print(
-                    f"DEBUG: Blockchain cancellation exception: {type(e).__name__}: {str(e)}"
-                )
-                if hasattr(e, "__dict__"):
-                    print(f"DEBUG: Exception attributes: {e.__dict__}")
-                result["blockchain_error"] = str(e)
+                    print(f"Warning: Error cancelling from blockchain: {error_str}")
+                    result["blockchain_result"] = {"success": False, "error": error_str}
 
-        # Calculate timing
+        # Update timing information
         result["timing"]["end_time"] = time.time()
         result["timing"]["duration_seconds"] = (
             result["timing"]["end_time"] - result["timing"]["start_time"]
@@ -1623,7 +1709,7 @@ class IPFSClient:
         metadata_cid: str,
         cancel_from_blockchain: bool = True,
         parallel_limit: int = 20,
-    ) -> Dict[str, Any]:
+    ) -> bool:
         """
         Delete an erasure-coded file, including all its chunks in parallel.
 
@@ -1633,219 +1719,84 @@ class IPFSClient:
             parallel_limit: Maximum number of concurrent deletion operations
 
         Returns:
-            Dict containing the result of the operation
+            bool: True if the deletion was successful, False otherwise
         """
-        result = {
-            "metadata_cid": metadata_cid,
-            "deleted_chunks": [],
-            "failed_chunks": [],
-            "blockchain_result": None,
-            "timing": {
-                "start_time": time.time(),
-                "end_time": None,
-                "duration_seconds": None,
-            },
-        }
 
-        # Track deletions for reporting
-        deleted_chunks_lock = asyncio.Lock()
-        failed_chunks_lock = asyncio.Lock()
+        # Try to download and process metadata file and chunks
+        ipfs_failure = False
+        metadata_error = False
 
-        # First, get the metadata to find all chunks
         try:
-            print(f"Downloading metadata file (CID: {metadata_cid})...")
-            start_time = time.time()
-            metadata_content = await self.client.cat(metadata_cid)
-            metadata = json.loads(metadata_content.decode("utf-8"))
-            metadata_download_time = time.time() - start_time
-
-            print(f"Metadata downloaded in {metadata_download_time:.2f} seconds")
-
-            # Extract chunk CIDs
-            chunks = []
-            total_chunks = 0
-
-            for chunk_data in metadata.get("chunks", []):
-                for ec_chunk in chunk_data.get("ec_chunks", []):
-                    chunk_cid = ec_chunk.get("cid")
-                    if chunk_cid:
-                        chunks.append(chunk_cid)
-                        total_chunks += 1
-
-            print(f"Found {total_chunks} chunks to delete")
+            # First download the metadata to get chunk CIDs
+            try:
+                metadata_result = await self.cat(metadata_cid)
+                metadata_json = json.loads(metadata_result["content"].decode("utf-8"))
+                chunks = metadata_json.get("chunks", [])
+            except json.JSONDecodeError:
+                # If we can't parse the metadata JSON, record the error but continue
+                metadata_error = True
+                # Continue with empty chunks so we can at least try to unpin the metadata file
+                chunks = []
+            except Exception:
+                # Any other metadata error
+                metadata_error = True
+                chunks = []
+
+            # Extract all chunk CIDs
+            chunk_cids = []
+            for chunk in chunks:
+                chunk_cid = chunk.get("cid", {})
+                if isinstance(chunk_cid, dict) and "cid" in chunk_cid:
+                    chunk_cids.append(chunk_cid["cid"])
+                elif isinstance(chunk_cid, str):
+                    chunk_cids.append(chunk_cid)
 
             # Create a semaphore to limit concurrent operations
-            sem = asyncio.Semaphore(parallel_limit)
+            semaphore = asyncio.Semaphore(parallel_limit)
 
-            # Define the chunk deletion function
-            async def delete_chunk(chunk_cid):
-                async with sem:
+            # Define the unpin task for each chunk with error handling
+            async def unpin_chunk(cid):
+                async with semaphore:
                     try:
-                        print(f"Unpinning chunk: {chunk_cid}")
-                        await self.client.unpin(chunk_cid)
-
-                        # Record success
-                        async with deleted_chunks_lock:
-                            result["deleted_chunks"].append(chunk_cid)
-
-                        # Cancel from blockchain if requested
-                        if cancel_from_blockchain:
-                            try:
-                                substrate_client = SubstrateClient()
-                                tx_hash = await substrate_client.cancel_storage_request(
-                                    chunk_cid
-                                )
-
-                                # Add blockchain result
-                                if "chunk_results" not in result["blockchain_result"]:
-                                    result["blockchain_result"] = {}
-                                    result["blockchain_result"]["chunk_results"] = []
-
-                                # Handle special return values from cancel_storage_request
-                                if tx_hash == "no-blockchain-cancellation-available":
-                                    result["blockchain_result"]["chunk_results"].append(
-                                        {
-                                            "cid": chunk_cid,
-                                            "status": "not_available",
-                                            "message": "Blockchain cancellation not available",
-                                        }
-                                    )
-                                elif tx_hash.startswith("ipfs-unpinned-only"):
-                                    error_msg = tx_hash.replace(
-                                        "ipfs-unpinned-only-", ""
-                                    )
-                                    result["blockchain_result"]["chunk_results"].append(
-                                        {
-                                            "cid": chunk_cid,
-                                            "status": "failed",
-                                            "error": error_msg,
-                                        }
-                                    )
-                                else:
-                                    # Standard successful transaction
-                                    result["blockchain_result"]["chunk_results"].append(
-                                        {
-                                            "cid": chunk_cid,
-                                            "transaction_hash": tx_hash,
-                                            "status": "success",
-                                        }
-                                    )
-                            except Exception as e:
-                                print(
-                                    f"Warning: Failed to cancel blockchain storage for chunk {chunk_cid}: {e}"
-                                )
-
-                                if "chunk_results" not in result["blockchain_result"]:
-                                    result["blockchain_result"] = {}
-                                    result["blockchain_result"]["chunk_results"] = []
-
-                                result["blockchain_result"]["chunk_results"].append(
-                                    {
-                                        "cid": chunk_cid,
-                                        "error": str(e),
-                                        "status": "failed",
-                                    }
-                                )
-
-                        return True
-                    except Exception as e:
-                        error_msg = f"Failed to delete chunk {chunk_cid}: {e}"
-                        print(f"Warning: {error_msg}")
-
-                        # Record failure
-                        async with failed_chunks_lock:
-                            result["failed_chunks"].append(
-                                {"cid": chunk_cid, "error": str(e)}
-                            )
-
-                        return False
-
-            # Start deleting chunks in parallel
-            print(
-                f"Starting parallel deletion of {total_chunks} chunks with max {parallel_limit} concurrent operations"
-            )
-            delete_tasks = [delete_chunk(cid) for cid in chunks]
-            await asyncio.gather(*delete_tasks)
-
-            # Delete the metadata file itself
-            print(f"Unpinning metadata file: {metadata_cid}")
-            response = await self.client.unpin(metadata_cid)
-
-            print(">>>", response)
-            raise SystemExit
-
-            # Cancel metadata from blockchain if requested
-            if cancel_from_blockchain:
-                try:
-                    print(f"Canceling blockchain storage request for metadata file...")
-                    substrate_client = SubstrateClient()
-                    tx_hash = await substrate_client.cancel_storage_request(
-                        metadata_cid
-                    )
-
-                    # Handle special return values from cancel_storage_request
-                    if tx_hash == "no-blockchain-cancellation-available":
-                        print(
-                            "Blockchain cancellation not available for metadata, but IPFS unpinning was successful"
-                        )
-                        result["blockchain_result"] = {
-                            "status": "not_available",
-                            "message": "Blockchain cancellation not available, but IPFS unpinning was successful",
-                        }
-                    elif tx_hash.startswith("ipfs-unpinned-only"):
-                        error_msg = tx_hash.replace("ipfs-unpinned-only-", "")
-                        print(
-                            f"IPFS unpinning successful, but blockchain cancellation failed for metadata: {error_msg}"
-                        )
-                        result["blockchain_result"] = {
-                            "status": "failed",
-                            "error": error_msg,
-                            "message": "IPFS unpinning successful, but blockchain cancellation failed",
-                        }
-                    else:
-                        # Standard successful transaction
-                        result["blockchain_result"] = {
-                            "metadata_transaction_hash": tx_hash,
-                            "status": "success",
-                        }
-                        print(
-                            f"Successfully canceled blockchain storage for metadata file"
-                        )
-                except Exception as e:
-                    print(
-                        f"Warning: Failed to cancel blockchain storage for metadata file: {e}"
-                    )
-
-                    if not result["blockchain_result"]:
-                        result["blockchain_result"] = {}
-
-                    result["blockchain_result"]["metadata_error"] = str(e)
-                    result["blockchain_result"]["status"] = "failed"
-
-            # Calculate and record timing information
-            end_time = time.time()
-            duration = end_time - result["timing"]["start_time"]
-
-            result["timing"]["end_time"] = end_time
-            result["timing"]["duration_seconds"] = duration
-
-            deleted_count = len(result["deleted_chunks"])
-            failed_count = len(result["failed_chunks"])
-
-            print(f"Deletion complete in {duration:.2f} seconds!")
-            print(f"Successfully deleted: {deleted_count}/{total_chunks} chunks")
-
-            if failed_count > 0:
-                print(f"Failed to delete: {failed_count}/{total_chunks} chunks")
+                        await self.client.unpin(cid)
+                        return {"success": True, "cid": cid}
+                    except Exception:
+                        # Record failure but continue with other chunks
+                        return {"success": False, "cid": cid}
+
+            # Unpin all chunks in parallel
+            if chunk_cids:
+                unpin_tasks = [unpin_chunk(cid) for cid in chunk_cids]
+                results = await asyncio.gather(*unpin_tasks)
+
+                # Count failures
+                failures = [r for r in results if not r["success"]]
+                if failures:
+                    ipfs_failure = True
+        except Exception:
+            # If we can't process chunks at all, record the failure
+            ipfs_failure = True
+
+        # Unpin the metadata file itself, regardless of whether we could process chunks
+        try:
+            await self.client.unpin(metadata_cid)
+        except Exception:
+            # Record the failure but continue with blockchain cancellation
+            ipfs_failure = True
 
-            return result
-        except Exception as e:
-            # Record end time even if there was an error
-            result["timing"]["end_time"] = time.time()
-            result["timing"]["duration_seconds"] = (
-                result["timing"]["end_time"] - result["timing"]["start_time"]
-            )
+        # Handle blockchain cancellation if requested
+        if cancel_from_blockchain:
+            # Create a substrate client
+            substrate_client = SubstrateClient()
 
-            error_msg = f"Error deleting erasure-coded file: {e}"
-            print(f"Error: {error_msg}")
-            raise RuntimeError(error_msg)
+            # This will raise appropriate exceptions if it fails:
+            # - HippiusAlreadyDeletedError if already deleted
+            # - HippiusFailedSubstrateDelete if transaction fails
+            # - Other exceptions for other failures
+            await substrate_client.cancel_storage_request(metadata_cid)
+
+        # If we get here, either:
+        # 1. Blockchain cancellation succeeded (if requested)
+        # 2. We weren't doing blockchain cancellation
+        # In either case, we report success
+        return True