hippius 0.1.0__py3-none-any.whl → 0.1.6__py3-none-any.whl

{hippius-0.1.0.dist-info → hippius-0.1.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hippius
-Version: 0.1.0
+Version: 0.1.6
 Summary: Python SDK and CLI for Hippius blockchain storage
 Home-page: https://github.com/thenervelab/hippius-sdk
 Author: Dubs
@@ -23,6 +23,7 @@ Requires-Dist: pyperclip (>=1.8.2,<2.0.0) ; extra == "clipboard"
 Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)
 Requires-Dist: requests (>=2.28.1,<3.0.0)
 Requires-Dist: substrate-interface (>=1.4.2,<2.0.0)
+Requires-Dist: zfec (>=1.5.3,<2.0.0)
 Project-URL: Documentation, https://github.com/thenervelab/hippius-sdk/docs
 Project-URL: Repository, https://github.com/thenervelab/hippius-sdk
 Description-Content-Type: text/markdown
@@ -172,6 +173,104 @@ raw_result = client.download_file(encrypted_result['cid'], "still_encrypted.txt"
 content = client.cat(encrypted_result['cid'], decrypt=True)
 ```
 
+## Erasure Coding
+
+Hippius SDK supports Reed-Solomon erasure coding for reliable and resilient file storage. This allows files to be split into chunks with added redundancy, so that the original file can be reconstructed even if some chunks are lost.
+
+### Erasure Coding Concepts
+
+- **k**: The number of data chunks needed to reconstruct the original file
+- **m**: The total number of chunks created (m > k)
+- The file can be reconstructed from any k chunks out of m total chunks
+- Higher redundancy (m-k) provides better protection against chunk loss
+
+### Using Erasure Coding
+
+```python
+from hippius_sdk import HippiusClient
+
+client = HippiusClient()
+
+# Erasure code a file with default parameters (k=3, m=5)
+result = client.erasure_code_file("large_file.mp4")
+metadata_cid = result["metadata_cid"]
+
+# Use custom parameters for more redundancy
+result = client.erasure_code_file(
+    file_path="important_data.zip",
+    k=4,               # Need 4 chunks to reconstruct
+    m=10,              # Create 10 chunks total (6 redundant)
+    chunk_size=2097152,  # 2MB chunks
+    encrypt=True       # Encrypt before splitting
+)
+
+# Store erasure-coded file in Hippius marketplace
+result = client.store_erasure_coded_file(
+    file_path="critical_backup.tar",
+    k=3,
+    m=5,
+    encrypt=True,
+    miner_ids=["miner1", "miner2", "miner3"]
+)
+
+# Reconstruct a file from its metadata
+reconstructed_path = client.reconstruct_from_erasure_code(
+    metadata_cid=metadata_cid,
+    output_file="reconstructed_file.mp4"
+)
+```
+
+### When to Use Erasure Coding
+
+Erasure coding is particularly useful for:
+
+- Large files where reliability is critical
+- Long-term archival storage
+- Data that must survive partial network failures
+- Situations where higher redundancy is needed without full replication
+
+### Advanced Features
+
+#### Small File Handling
+
+The SDK automatically adjusts parameters for small files:
+
+- If a file is too small to be split into `k` chunks, the SDK will adjust the chunk size
+- For very small files, the content is split into exactly `k` sub-blocks
+- Parameters are always optimized to provide the requested level of redundancy
+
+#### Robust Storage in Marketplace
+
+When using `store_erasure_coded_file`, the SDK now:
+
+- Stores both the metadata file AND all encoded chunks in the marketplace
+- Ensures miners can access all necessary data for redundancy and retrieval
+- Reports total number of files stored for verification
+
+#### CLI Commands
+
+The CLI provides powerful commands for erasure coding:
+
+```bash
+# Basic usage with automatic parameter adjustment
+hippius erasure-code myfile.txt
+
+# Specify custom parameters
+hippius erasure-code large_video.mp4 --k 4 --m 8 --chunk-size 4194304
+
+# For smaller files, using smaller parameters
+hippius erasure-code small_doc.txt --k 2 --m 5 --chunk-size 4096
+
+# Reconstruct a file from its metadata CID
+hippius reconstruct QmMetadataCID reconstructed_file.mp4
+```
+
+The CLI provides detailed output during the process, including:
+- Automatic parameter adjustments for optimal encoding
+- Progress of chunk creation and upload
+- Storage confirmation in the marketplace
+- Instructions for reconstruction
+
 ## Command Line Interface
 
 The Hippius SDK includes a powerful command-line interface (CLI) that provides access to all major features of the SDK directly from your terminal.
@@ -240,6 +339,19 @@ hippius store my_file.txt --encrypt
 hippius download QmCID123 output_file.txt --decrypt
 ```
 
+### Erasure Coding
+
+```bash
+# Erasure code a file with default parameters (k=3, m=5)
+hippius erasure-code large_file.mp4
+
+# Erasure code with custom parameters
+hippius erasure-code important_data.zip --k 4 --m 10 --chunk-size 2097152 --encrypt
+
+# Reconstruct a file from its metadata
+hippius reconstruct QmMetadataCID reconstructed_file.mp4
+```
+
 ### Using Environment Variables
 
 The CLI automatically reads from your `.env` file for common settings:

hippius-0.1.6.dist-info/RECORD

@@ -0,0 +1,9 @@
+hippius_sdk/__init__.py,sha256=SwOREu9EJZ9ZRM-rSPX0o1hhsOUIADuP3CxoF4Mp_qI,288
+hippius_sdk/cli.py,sha256=WfjU9nuUBXN6Tu25PnOpLHftClP_umh6Zl9t4BOzAfo,30576
+hippius_sdk/client.py,sha256=bHsoadw2WMMZDU7D0r02nHeU82PAa4cvmblieDzBw54,13305
+hippius_sdk/ipfs.py,sha256=C9oMTBefCIfWFUsUBxhUkivz5rIUkhHKJtqdVIkMAbc,61475
+hippius_sdk/substrate.py,sha256=mfDxbKn9HdtcK1xEnj_BnnreRw8ITZswtDoBhtliidM,27278
+hippius-0.1.6.dist-info/METADATA,sha256=295Uv9mZq1G0pypT4PibEmTDVNRr7gM_ScFNVPZTfdo,16580
+hippius-0.1.6.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+hippius-0.1.6.dist-info/entry_points.txt,sha256=b1lo60zRXmv1ud-c5BC-cJcAfGE5FD4qM_nia6XeQtM,98
+hippius-0.1.6.dist-info/RECORD,,

hippius_sdk/cli.py

@@ -435,25 +435,202 @@ def handle_files(client, account_address, debug=False, show_all_miners=False):
     return 0
 
 
+def handle_erasure_code(
+    client, file_path, k, m, chunk_size, miner_ids, encrypt=None, verbose=True
+):
+    """Handle the erasure-code command"""
+    if not os.path.exists(file_path):
+        print(f"Error: File {file_path} not found")
+        return 1
+
+    # Check if zfec is installed
+    try:
+        import zfec
+    except ImportError:
+        print(
+            "Error: zfec is required for erasure coding. Install it with: pip install zfec"
+        )
+        print("Then update your environment: poetry add zfec")
+        return 1
+
+    # Parse miner IDs if provided
+    miner_id_list = None
+    if miner_ids:
+        miner_id_list = [m.strip() for m in miner_ids.split(",") if m.strip()]
+        if verbose:
+            print(f"Targeting {len(miner_id_list)} miners: {', '.join(miner_id_list)}")
+
+    # Get the file size and adjust parameters if needed
+    file_size = os.path.getsize(file_path)
+    file_size_mb = file_size / (1024 * 1024)
+    
+    print(f"Processing {file_path} ({file_size_mb:.2f} MB) with erasure coding...")
+    
+    # Check if the file is too small for the current chunk size and k value
+    original_k = k
+    original_m = m
+    original_chunk_size = chunk_size
+    
+    # Calculate how many chunks we would get with current settings
+    potential_chunks = max(1, file_size // chunk_size)
+    
+    # If we can't get at least k chunks, adjust the chunk size
+    if potential_chunks < k:
+        # Calculate a new chunk size that would give us exactly k chunks
+        new_chunk_size = max(1024, file_size // k)  # Ensure at least 1KB chunks
+        
+        print(f"Warning: File is too small for the requested parameters.")
+        print(f"Original parameters: k={k}, m={m}, chunk size={chunk_size/1024/1024:.2f} MB")
+        print(f"Would create only {potential_chunks} chunks, which is less than k={k}")
+        print(f"Automatically adjusting chunk size to {new_chunk_size/1024/1024:.6f} MB to create at least {k} chunks")
+        
+        chunk_size = new_chunk_size
+    
+    print(f"Final parameters: k={k}, m={m} (need {k} of {m} chunks to reconstruct)")
+    print(f"Chunk size: {chunk_size/1024/1024:.6f} MB")
+
+    if encrypt:
+        print("Encryption: Enabled")
+
+    start_time = time.time()
+
+    try:
+        # Use the store_erasure_coded_file method directly from HippiusClient
+        result = client.store_erasure_coded_file(
+            file_path=file_path,
+            k=k,
+            m=m,
+            chunk_size=chunk_size,
+            encrypt=encrypt,
+            miner_ids=miner_id_list,
+            max_retries=3,
+            verbose=verbose,
+        )
+
+        elapsed_time = time.time() - start_time
+
+        print(f"\nErasure coding and storage completed in {elapsed_time:.2f} seconds!")
+
+        # Display metadata
+        metadata = result.get("metadata", {})
+        metadata_cid = result.get("metadata_cid", "unknown")
+        total_files_stored = result.get("total_files_stored", 0)
+
+        original_file = metadata.get("original_file", {})
+        erasure_coding = metadata.get("erasure_coding", {})
+
+        print("\nErasure Coding Summary:")
+        print(
+            f"  Original file: {original_file.get('name')} ({original_file.get('size', 0)/1024/1024:.2f} MB)"
+        )
+        print(f"  File ID: {erasure_coding.get('file_id')}")
+        print(f"  Parameters: k={erasure_coding.get('k')}, m={erasure_coding.get('m')}")
+        print(f"  Total chunks: {len(metadata.get('chunks', []))}")
+        print(f"  Total files stored in marketplace: {total_files_stored}")
+        print(f"  Metadata CID: {metadata_cid}")
+
+        # If we stored in the marketplace
+        if "transaction_hash" in result:
+            print(
+                f"\nStored in marketplace. Transaction hash: {result['transaction_hash']}"
+            )
+
+        # Instructions for reconstruction
+        print("\nTo reconstruct this file, you will need:")
+        print(f"  1. The metadata CID: {metadata_cid}")
+        print("  2. Access to at least k chunks for each original chunk")
+        print("\nReconstruction command:")
+        print(
+            f"  hippius reconstruct {metadata_cid} reconstructed_{original_file.get('name')}"
+        )
+
+        return 0
+
+    except Exception as e:
+        print(f"Error during erasure coding: {e}")
+        
+        # Provide helpful advice based on the error
+        if "Wrong length" in str(e) and "input blocks" in str(e):
+            print("\nThis error typically occurs with very small files.")
+            print("Suggestions:")
+            print("  1. Try using a smaller chunk size: --chunk-size 4096")
+            print("  2. Try using a smaller k value: --k 2")
+            print("  3. For very small files, consider using regular storage instead of erasure coding.")
+        
+        return 1
+
+
+def handle_reconstruct(client, metadata_cid, output_file, verbose=True):
+    """Handle the reconstruct command for erasure-coded files"""
+    # Check if zfec is installed
+    try:
+        import zfec
+    except ImportError:
+        print(
+            "Error: zfec is required for erasure coding. Install it with: pip install zfec"
+        )
+        print("Then update your environment: poetry add zfec")
+        return 1
+
+    print(f"Reconstructing file from metadata CID: {metadata_cid}")
+    print(f"Output file: {output_file}")
+
+    start_time = time.time()
+
+    try:
+        # Use the reconstruct_from_erasure_code method
+        result = client.reconstruct_from_erasure_code(
+            metadata_cid=metadata_cid, output_file=output_file, verbose=verbose
+        )
+
+        elapsed_time = time.time() - start_time
+        print(f"\nFile reconstruction completed in {elapsed_time:.2f} seconds!")
+        print(f"Reconstructed file saved to: {result}")
+
+        return 0
+
+    except Exception as e:
+        print(f"Error during file reconstruction: {e}")
+        return 1
+
+
 def main():
-    """Main entry point for the Hippius CLI."""
+    """Main CLI entry point for hippius command."""
+    # Set up the argument parser
     parser = argparse.ArgumentParser(
         description="Hippius SDK Command Line Interface",
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog="""
-Examples:
-  hippius download QmCID123 downloaded_file.txt
-  hippius exists QmCID123
-  hippius cat QmCID123
-  hippius store test_file.txt
-  hippius store-dir ./test_directory
+examples:
+  # Store a file
+  hippius store example.txt
+  
+  # Store a directory
+  hippius store-dir ./my_directory
+  
+  # Download a file
+  hippius download QmHash output.txt
+  
+  # Check if a CID exists
+  hippius exists QmHash
+  
+  # View the content of a CID
+  hippius cat QmHash
+  
+  # View your available credits
   hippius credits
-  hippius credits 5H1QBRF7T7dgKwzVGCgS4wioudvMRf9K4NEDzfuKLnuyBNzH
+  
+  # View your stored files
   hippius files
-  hippius files 5H1QBRF7T7dgKwzVGCgS4wioudvMRf9K4NEDzfuKLnuyBNzH
+  
+  # View all miners for stored files
   hippius files --all-miners
-  hippius keygen
-  hippius keygen --copy
+  
+  # Erasure code a file (Reed-Solomon)
+  hippius erasure-code large_file.mp4 --k 3 --m 5
+  
+  # Reconstruct an erasure-coded file
+  hippius reconstruct QmMetadataHash reconstructed_file.mp4
 """,
     )
 
@@ -588,6 +765,53 @@ Examples:
         "--copy", action="store_true", help="Copy the generated key to the clipboard"
     )
 
+    # Erasure code command
+    erasure_code_parser = subparsers.add_parser(
+        "erasure-code", help="Erasure code a file"
+    )
+    erasure_code_parser.add_argument("file_path", help="Path to file to erasure code")
+    erasure_code_parser.add_argument(
+        "--k",
+        type=int,
+        default=3,
+        help="Number of data chunks needed to reconstruct (default: 3)",
+    )
+    erasure_code_parser.add_argument(
+        "--m", type=int, default=5, help="Total number of chunks to create (default: 5)"
+    )
+    erasure_code_parser.add_argument(
+        "--chunk-size",
+        type=int,
+        default=1048576,
+        help="Chunk size in bytes (default: 1MB)",
+    )
+    erasure_code_parser.add_argument(
+        "--miner-ids", help="Comma-separated list of miner IDs"
+    )
+    erasure_code_parser.add_argument(
+        "--encrypt", action="store_true", help="Encrypt the file"
+    )
+    erasure_code_parser.add_argument(
+        "--no-encrypt", action="store_true", help="Do not encrypt the file"
+    )
+    erasure_code_parser.add_argument(
+        "--verbose", action="store_true", help="Enable verbose output", default=True
+    )
+
+    # Reconstruct command
+    reconstruct_parser = subparsers.add_parser(
+        "reconstruct", help="Reconstruct an erasure-coded file"
+    )
+    reconstruct_parser.add_argument(
+        "metadata_cid", help="Metadata CID of the erasure-coded file"
+    )
+    reconstruct_parser.add_argument(
+        "output_file", help="Path to save reconstructed file"
+    )
+    reconstruct_parser.add_argument(
+        "--verbose", action="store_true", help="Enable verbose output", default=True
+    )
+
     args = parser.parse_args()
 
     if not args.command:
@@ -647,6 +871,23 @@ Examples:
                 else False,
             )
 
+        elif args.command == "erasure-code":
+            return handle_erasure_code(
+                client,
+                args.file_path,
+                args.k,
+                args.m,
+                args.chunk_size,
+                miner_ids,
+                encrypt=args.encrypt,
+                verbose=args.verbose,
+            )
+
+        elif args.command == "reconstruct":
+            return handle_reconstruct(
+                client, args.metadata_cid, args.output_file, verbose=args.verbose
+            )
+
     except Exception as e:
         print(f"Error: {e}")
         return 1

hippius_sdk/client.py

@@ -35,7 +35,7 @@ class HippiusClient:
             encrypt_by_default: Whether to encrypt files by default (from .env if None)
             encryption_key: Encryption key for NaCl secretbox (from .env if None)
         """
-        self.ipfs = IPFSClient(
+        self.ipfs_client = IPFSClient(
             gateway=ipfs_gateway,
             api_url=ipfs_api_url,
             encrypt_by_default=encrypt_by_default,
@@ -75,7 +75,7 @@ class HippiusClient:
             ValueError: If encryption is requested but not available
         """
         # Use the enhanced IPFSClient method directly with encryption parameter
-        return self.ipfs.upload_file(file_path, encrypt=encrypt)
+        return self.ipfs_client.upload_file(file_path, encrypt=encrypt)
 
     def upload_directory(
         self, dir_path: str, encrypt: Optional[bool] = None
@@ -102,7 +102,7 @@ class HippiusClient:
             ValueError: If encryption is requested but not available
         """
         # Use the enhanced IPFSClient method directly with encryption parameter
-        return self.ipfs.upload_directory(dir_path, encrypt=encrypt)
+        return self.ipfs_client.upload_directory(dir_path, encrypt=encrypt)
 
     def download_file(
         self, cid: str, output_path: str, decrypt: Optional[bool] = None
@@ -128,7 +128,7 @@ class HippiusClient:
             requests.RequestException: If the download fails
             ValueError: If decryption is requested but fails
         """
-        return self.ipfs.download_file(cid, output_path, decrypt=decrypt)
+        return self.ipfs_client.download_file(cid, output_path, decrypt=decrypt)
 
     def cat(
         self,
@@ -155,7 +155,9 @@ class HippiusClient:
                 - text_preview/hex_preview: Preview of the content
                 - decrypted: Whether the file was decrypted
         """
-        return self.ipfs.cat(cid, max_display_bytes, format_output, decrypt=decrypt)
+        return self.ipfs_client.cat(
+            cid, max_display_bytes, format_output, decrypt=decrypt
+        )
 
     def exists(self, cid: str) -> Dict[str, Any]:
         """
@@ -171,7 +173,7 @@ class HippiusClient:
                 - formatted_cid: Formatted version of the CID
                 - gateway_url: URL to access the content if it exists
         """
-        return self.ipfs.exists(cid)
+        return self.ipfs_client.exists(cid)
 
     def pin(self, cid: str) -> Dict[str, Any]:
         """
@@ -187,7 +189,7 @@ class HippiusClient:
                 - formatted_cid: Formatted version of the CID
                 - message: Status message
         """
-        return self.ipfs.pin(cid)
+        return self.ipfs_client.pin(cid)
 
     def format_cid(self, cid: str) -> str:
         """
@@ -201,7 +203,7 @@ class HippiusClient:
         Returns:
             str: Formatted CID string
         """
-        return self.ipfs.format_cid(cid)
+        return self.ipfs_client.format_cid(cid)
 
     def format_size(self, size_bytes: int) -> str:
         """
@@ -215,7 +217,7 @@ class HippiusClient:
         Returns:
             str: Human-readable size string (e.g., '1.23 MB', '456.78 KB')
         """
-        return self.ipfs.format_size(size_bytes)
+        return self.ipfs_client.format_size(size_bytes)
 
     def generate_encryption_key(self) -> str:
         """
@@ -244,3 +246,125 @@ class HippiusClient:
             raise ImportError(
                 "PyNaCl is required for encryption. Install it with: pip install pynacl"
             )
+
+    def erasure_code_file(
+        self,
+        file_path: str,
+        k: int = 3,
+        m: int = 5,
+        chunk_size: int = 1024 * 1024,  # 1MB chunks
+        encrypt: Optional[bool] = None,
+        max_retries: int = 3,
+        verbose: bool = True,
+    ) -> Dict[str, Any]:
+        """
+        Split a file using erasure coding, then upload the chunks to IPFS.
+
+        This implements an (m, k) Reed-Solomon code where:
+        - m = total number of chunks
+        - k = minimum chunks needed to reconstruct the file (k <= m)
+        - The file can be reconstructed from any k of the m chunks
+
+        Args:
+            file_path: Path to the file to upload
+            k: Number of data chunks (minimum required to reconstruct)
+            m: Total number of chunks (k + redundancy)
+            chunk_size: Size of each chunk in bytes before encoding
+            encrypt: Whether to encrypt the file before encoding (defaults to self.encrypt_by_default)
+            max_retries: Maximum number of retry attempts for IPFS uploads
+            verbose: Whether to print progress information
+
+        Returns:
+            dict: Metadata including the original file info and chunk information
+
+        Raises:
+            ValueError: If erasure coding is not available or parameters are invalid
+            RuntimeError: If chunk uploads fail
+        """
+        return self.ipfs_client.erasure_code_file(
+            file_path=file_path,
+            k=k,
+            m=m,
+            chunk_size=chunk_size,
+            encrypt=encrypt,
+            max_retries=max_retries,
+            verbose=verbose,
+        )
+
+    def reconstruct_from_erasure_code(
+        self,
+        metadata_cid: str,
+        output_file: str,
+        temp_dir: str = None,
+        max_retries: int = 3,
+        verbose: bool = True,
+    ) -> str:
+        """
+        Reconstruct a file from erasure-coded chunks using its metadata.
+
+        Args:
+            metadata_cid: IPFS CID of the metadata file
+            output_file: Path where the reconstructed file should be saved
+            temp_dir: Directory to use for temporary files (default: system temp)
+            max_retries: Maximum number of retry attempts for IPFS downloads
+            verbose: Whether to print progress information
+
+        Returns:
+            str: Path to the reconstructed file
+
+        Raises:
+            ValueError: If reconstruction fails
+            RuntimeError: If not enough chunks can be downloaded
+        """
+        return self.ipfs_client.reconstruct_from_erasure_code(
+            metadata_cid=metadata_cid,
+            output_file=output_file,
+            temp_dir=temp_dir,
+            max_retries=max_retries,
+            verbose=verbose,
+        )
+
+    def store_erasure_coded_file(
+        self,
+        file_path: str,
+        k: int = 3,
+        m: int = 5,
+        chunk_size: int = 1024 * 1024,  # 1MB chunks
+        encrypt: Optional[bool] = None,
+        miner_ids: List[str] = None,
+        max_retries: int = 3,
+        verbose: bool = True,
+    ) -> Dict[str, Any]:
+        """
+        Erasure code a file, upload the chunks to IPFS, and store in the Hippius marketplace.
+
+        This is a convenience method that combines erasure_code_file with storage_request.
+
+        Args:
+            file_path: Path to the file to upload
+            k: Number of data chunks (minimum required to reconstruct)
+            m: Total number of chunks (k + redundancy)
+            chunk_size: Size of each chunk in bytes before encoding
+            encrypt: Whether to encrypt the file before encoding
+            miner_ids: List of specific miner IDs to use for storage
+            max_retries: Maximum number of retry attempts
+            verbose: Whether to print progress information
+
+        Returns:
+            dict: Result including metadata CID and transaction hash
+
+        Raises:
+            ValueError: If parameters are invalid
+            RuntimeError: If processing fails
+        """
+        return self.ipfs_client.store_erasure_coded_file(
+            file_path=file_path,
+            k=k,
+            m=m,
+            chunk_size=chunk_size,
+            encrypt=encrypt,
+            miner_ids=miner_ids,
+            substrate_client=self.substrate_client,
+            max_retries=max_retries,
+            verbose=verbose,
+        )

hippius_sdk/ipfs.py

@@ -8,7 +8,9 @@ import requests
 import base64
 import time
 import tempfile
-from typing import Dict, Any, Optional, Union, List
+import hashlib
+import uuid
+from typing import Dict, Any, Optional, Union, List, Tuple
 import ipfshttpclient
 from dotenv import load_dotenv
 
@@ -21,6 +23,14 @@ try:
 except ImportError:
     ENCRYPTION_AVAILABLE = False
 
+# Import zfec for erasure coding
+try:
+    import zfec
+
+    ERASURE_CODING_AVAILABLE = True
+except ImportError:
+    ERASURE_CODING_AVAILABLE = False
+
 
 class IPFSClient:
     """Client for interacting with IPFS."""
@@ -288,6 +298,7 @@ class IPFSClient:
         file_path: str,
         include_formatted_size: bool = True,
         encrypt: Optional[bool] = None,
+        max_retries: int = 3,
     ) -> Dict[str, Any]:
         """
         Upload a file to IPFS with optional encryption.
@@ -296,6 +307,7 @@ class IPFSClient:
             file_path: Path to the file to upload
             include_formatted_size: Whether to include formatted size in the result (default: True)
             encrypt: Whether to encrypt the file (overrides default)
+            max_retries: Maximum number of retry attempts (default: 3)
 
         Returns:
             Dict[str, Any]: Dictionary containing:
@@ -355,7 +367,7 @@ class IPFSClient:
                 cid = result["Hash"]
             elif self.base_url:
                 # Fallback to using HTTP API
-                cid = self._upload_via_http_api(upload_path)
+                cid = self._upload_via_http_api(upload_path, max_retries=max_retries)
             else:
                 # No connection or API URL available
                 raise ConnectionError(
@@ -983,3 +995,577 @@ class IPFSClient:
             "formatted_cid": formatted_cid,
             "message": message,
         }
+
+    def erasure_code_file(
+        self,
+        file_path: str,
+        k: int = 3,
+        m: int = 5,
+        chunk_size: int = 1024 * 1024,  # 1MB chunks
+        encrypt: Optional[bool] = None,
+        max_retries: int = 3,
+        verbose: bool = True,
+    ) -> Dict[str, Any]:
+        """
+        Split a file using erasure coding, then upload the chunks to IPFS.
+
+        This implements an (m, k) Reed-Solomon code where:
+        - m = total number of chunks
+        - k = minimum chunks needed to reconstruct the file (k <= m)
+        - The file can be reconstructed from any k of the m chunks
+
+        Args:
+            file_path: Path to the file to upload
+            k: Number of data chunks (minimum required to reconstruct)
+            m: Total number of chunks (k + redundancy)
+            chunk_size: Size of each chunk in bytes before encoding
+            encrypt: Whether to encrypt the file before encoding (defaults to self.encrypt_by_default)
+            max_retries: Maximum number of retry attempts for IPFS uploads
+            verbose: Whether to print progress information
+
+        Returns:
+            dict: Metadata including the original file info and chunk information
+
+        Raises:
+            ValueError: If erasure coding is not available or parameters are invalid
+            RuntimeError: If chunk uploads fail
+        """
+        if not ERASURE_CODING_AVAILABLE:
+            raise ValueError(
+                "Erasure coding is not available. Install zfec: pip install zfec"
+            )
+
+        if k >= m:
+            raise ValueError(
+                f"Invalid erasure coding parameters: k ({k}) must be less than m ({m})"
+            )
+
+        # Get original file info
+        file_name = os.path.basename(file_path)
+        file_size = os.path.getsize(file_path)
+        file_extension = os.path.splitext(file_name)[1]
+
+        # Determine if encryption should be used
+        should_encrypt = self.encrypt_by_default if encrypt is None else encrypt
+
+        if should_encrypt and not self.encryption_available:
+            raise ValueError(
+                "Encryption requested but not available. Install PyNaCl and configure an encryption key."
+            )
+
+        # Generate a unique ID for this file
+        file_id = str(uuid.uuid4())
+
+        if verbose:
+            print(f"Processing file: {file_name} ({file_size/1024/1024:.2f} MB)")
+            print(
+                f"Erasure coding parameters: k={k}, m={m} (need {k}/{m} chunks to reconstruct)"
+            )
+            if should_encrypt:
+                print("Encryption: Enabled")
+
+        # Step 1: Read and potentially encrypt the file
+        with open(file_path, "rb") as f:
+            file_data = f.read()
+
+        # Calculate original file hash
+        original_file_hash = hashlib.sha256(file_data).hexdigest()
+
+        # Encrypt if requested
+        if should_encrypt:
+            if verbose:
+                print("Encrypting file data...")
+            file_data = self.encrypt_data(file_data)
+
+        # Step 2: Split the file into chunks for erasure coding
+        chunks = []
+        chunk_positions = []
+        for i in range(0, len(file_data), chunk_size):
+            chunk = file_data[i : i + chunk_size]
+            chunks.append(chunk)
+            chunk_positions.append(i)
+
+        # Pad the last chunk if necessary
+        if chunks and len(chunks[-1]) < chunk_size:
+            pad_size = chunk_size - len(chunks[-1])
+            chunks[-1] = chunks[-1] + b"\0" * pad_size
+
+        # If we don't have enough chunks for the requested parameters, adjust
+        if len(chunks) < k:
+            if verbose:
+                print(
+                    f"Warning: File has fewer chunks ({len(chunks)}) than k={k}. Adjusting parameters."
+                )
+            
+            # If we have a very small file, we'll just use a single chunk
+            # but will still split it into k sub-blocks during encoding
+            if len(chunks) == 1:
+                if verbose:
+                    print(f"Small file (single chunk): will split into {k} sub-blocks for encoding")
+            else:
+                # If we have multiple chunks but fewer than k, adjust k to match
+                old_k = k
+                k = max(1, len(chunks))
+                if verbose:
+                    print(f"Adjusting k from {old_k} to {k} to match available chunks")
+                    
+            # Ensure m is greater than k for redundancy
+            if m <= k:
+                old_m = m
+                m = k + 2  # Ensure we have at least 2 redundant chunks
+                if verbose:
+                    print(f"Adjusting m from {old_m} to {m} to ensure redundancy")
+            
+            if verbose:
+                print(f"New parameters: k={k}, m={m}")
+                
+        # Ensure we have at least one chunk to process
+        if not chunks:
+            raise ValueError("File is empty or too small to process")
+            
+        # For k=1 case, ensure we have proper sized input for zfec
+        if k == 1 and len(chunks) == 1:
+            # zfec expects the input to be exactly chunk_size for k=1
+            # So we need to pad if shorter or truncate if longer
+            if len(chunks[0]) != chunk_size:
+                chunks[0] = chunks[0].ljust(chunk_size, b'\0')[:chunk_size]
+
+        # Create metadata
+        metadata = {
+            "original_file": {
+                "name": file_name,
+                "size": file_size,
+                "hash": original_file_hash,
+                "extension": file_extension,
+            },
+            "erasure_coding": {
+                "k": k,
+                "m": m,
+                "chunk_size": chunk_size,
+                "encrypted": should_encrypt,
+                "file_id": file_id,
+            },
+            "chunks": [],
+        }
+
+        # Step 3: Apply erasure coding to each chunk
+        if verbose:
+            print(f"Applying erasure coding to {len(chunks)} chunks...")
+
+        all_encoded_chunks = []
+        for i, chunk in enumerate(chunks):
+            try:
+                # For zfec encoder.encode(), we must provide exactly k blocks
+                
+                # Calculate how many bytes each sub-block should have
+                sub_block_size = (len(chunk) + k - 1) // k  # ceiling division for even distribution
+                
+                # Split the chunk into exactly k sub-blocks of equal size (padding as needed)
+                sub_blocks = []
+                for j in range(k):
+                    start = j * sub_block_size
+                    end = min(start + sub_block_size, len(chunk))
+                    sub_block = chunk[start:end]
+                    
+                    # Pad if needed to make all sub-blocks the same size
+                    if len(sub_block) < sub_block_size:
+                        sub_block = sub_block.ljust(sub_block_size, b'\0')
+                        
+                    sub_blocks.append(sub_block)
+                
+                # Verify we have exactly k sub-blocks
+                if len(sub_blocks) != k:
+                    raise ValueError(f"Expected {k} sub-blocks but got {len(sub_blocks)}")
+                
+                # Encode the k sub-blocks to create m encoded blocks
+                encoder = zfec.Encoder(k, m)
+                encoded_chunks = encoder.encode(sub_blocks)
+                
+                # Add to our collection
+                all_encoded_chunks.append(encoded_chunks)
+                
+                if verbose and (i + 1) % 10 == 0:
+                    print(f"  Encoded {i+1}/{len(chunks)} chunks")
+            except Exception as e:
+                # If encoding fails, provide more helpful error message
+                error_msg = f"Error encoding chunk {i}: {str(e)}"
+                print(f"Error details: chunk size={len(chunk)}, k={k}, m={m}")
+                print(f"Sub-blocks created: {len(sub_blocks) if 'sub_blocks' in locals() else 'None'}")
+                raise RuntimeError(f"{error_msg}")
+
+        # Step 4: Upload all chunks to IPFS
+        if verbose:
+            print(f"Uploading {len(chunks) * m} erasure-coded chunks to IPFS...")
+
+        chunk_uploads = 0
+        chunk_data = []
+
+        # Create a temporary directory for the chunks
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Write and upload each encoded chunk
+            for original_idx, encoded_chunks in enumerate(all_encoded_chunks):
+                for share_idx, share_data in enumerate(encoded_chunks):
+                    # Create a name for this chunk that includes needed info
+                    chunk_name = f"{file_id}_chunk_{original_idx}_{share_idx}.ec"
+                    chunk_path = os.path.join(temp_dir, chunk_name)
+
+                    # Write the chunk to a temp file
+                    with open(chunk_path, "wb") as f:
+                        f.write(share_data)
+
+                    # Upload the chunk to IPFS
+                    try:
+                        chunk_cid = self.upload_file(
+                            chunk_path, max_retries=max_retries
+                        )
+
+                        # Store info about this chunk
+                        chunk_info = {
+                            "name": chunk_name,
+                            "cid": chunk_cid,
+                            "original_chunk": original_idx,
+                            "share_idx": share_idx,
+                            "size": len(share_data),
+                        }
+                        chunk_data.append(chunk_info)
+
+                        chunk_uploads += 1
+                        if verbose and chunk_uploads % 10 == 0:
+                            print(
+                                f"  Uploaded {chunk_uploads}/{len(chunks) * m} chunks"
+                            )
+                    except Exception as e:
+                        print(f"Error uploading chunk {chunk_name}: {str(e)}")
+
+            # Add all chunk info to metadata
+            metadata["chunks"] = chunk_data
+
+            # Step 5: Create and upload the metadata file
+            metadata_path = os.path.join(temp_dir, f"{file_id}_metadata.json")
+            with open(metadata_path, "w") as f:
+                json.dump(metadata, f, indent=2)
+
+            if verbose:
+                print(f"Uploading metadata file...")
+
+            # Upload the metadata file to IPFS
+            metadata_cid_result = self.upload_file(metadata_path, max_retries=max_retries)
+            
+            # Extract just the CID string from the result dictionary
+            metadata_cid = metadata_cid_result['cid']
+            metadata["metadata_cid"] = metadata_cid
+
+            if verbose:
+                print(f"Erasure coding complete!")
+                print(f"Metadata CID: {metadata_cid}")
+                print(f"Original file size: {file_size/1024/1024:.2f} MB")
+                print(f"Total chunks: {len(chunks) * m}")
+                print(f"Minimum chunks needed: {k * len(chunks)}")
+
+            return metadata
+
+    def reconstruct_from_erasure_code(
+        self,
+        metadata_cid: str,
+        output_file: str,
+        temp_dir: str = None,
+        max_retries: int = 3,
+        verbose: bool = True,
+    ) -> str:
+        """
+        Reconstruct a file from erasure-coded chunks using its metadata.
+
+        Args:
+            metadata_cid: IPFS CID of the metadata file
+            output_file: Path where the reconstructed file should be saved
+            temp_dir: Directory to use for temporary files (default: system temp)
+            max_retries: Maximum number of retry attempts for IPFS downloads
+            verbose: Whether to print progress information
+
+        Returns:
+            str: Path to the reconstructed file
+
+        Raises:
+            ValueError: If reconstruction fails
+            RuntimeError: If not enough chunks can be downloaded
+        """
+        if not ERASURE_CODING_AVAILABLE:
+            raise ValueError(
+                "Erasure coding is not available. Install zfec: pip install zfec"
+            )
+
+        # Create a temporary directory if not provided
+        if temp_dir is None:
+            temp_dir_obj = tempfile.TemporaryDirectory()
+            temp_dir = temp_dir_obj.name
+        else:
+            temp_dir_obj = None
+
+        try:
+            # Step 1: Download and parse the metadata file
+            if verbose:
+                print(f"Downloading metadata file (CID: {metadata_cid})...")
+
+            metadata_path = os.path.join(temp_dir, "metadata.json")
+            self.download_file(metadata_cid, metadata_path, max_retries=max_retries)
+
+            with open(metadata_path, "r") as f:
+                metadata = json.load(f)
+
+            # Step 2: Extract key information
+            original_file = metadata["original_file"]
+            erasure_params = metadata["erasure_coding"]
+            chunks_info = metadata["chunks"]
+
+            k = erasure_params["k"]
+            m = erasure_params["m"]
+            is_encrypted = erasure_params.get("encrypted", False)
+            chunk_size = erasure_params.get("chunk_size", 1024 * 1024)
+
+            if verbose:
+                print(
+                    f"File: {original_file['name']} ({original_file['size']/1024/1024:.2f} MB)"
+                )
+                print(f"Erasure coding parameters: k={k}, m={m}")
+                print(f"Encrypted: {is_encrypted}")
+
+            # Step 3: Group chunks by their original chunk index
+            chunks_by_original = {}
+            for chunk in chunks_info:
+                orig_idx = chunk["original_chunk"]
+                if orig_idx not in chunks_by_original:
+                    chunks_by_original[orig_idx] = []
+                chunks_by_original[orig_idx].append(chunk)
+
+            # Step 4: For each original chunk, download at least k shares
+            if verbose:
+                print(f"Downloading and reconstructing chunks...")
+
+            reconstructed_chunks = []
+
+            for orig_idx in sorted(chunks_by_original.keys()):
+                available_chunks = chunks_by_original[orig_idx]
+
+                if len(available_chunks) < k:
+                    raise ValueError(
+                        f"Not enough chunks available for original chunk {orig_idx}. "
+                        f"Need {k}, but only have {len(available_chunks)}."
+                    )
+
+                # We only need k chunks, so take the first k
+                chunks_to_download = available_chunks[:k]
+
+                # Download the chunks
+                downloaded_shares = []
+                share_indexes = []
+
+                for chunk in chunks_to_download:
+                    chunk_path = os.path.join(temp_dir, chunk["name"])
+                    try:
+                        # Extract the CID string from the chunk's cid dictionary
+                        chunk_cid = chunk["cid"]["cid"] if isinstance(chunk["cid"], dict) and "cid" in chunk["cid"] else chunk["cid"]
+                        self.download_file(
+                            chunk_cid, chunk_path, max_retries=max_retries
+                        )
+
+                        # Read the chunk data
+                        with open(chunk_path, "rb") as f:
+                            share_data = f.read()
+
+                        downloaded_shares.append(share_data)
+                        share_indexes.append(chunk["share_idx"])
+
+                    except Exception as e:
+                        if verbose:
+                            print(f"Error downloading chunk {chunk['name']}: {str(e)}")
+                        # Continue to the next chunk
+
+                # If we don't have enough chunks, try to download more
+                if len(downloaded_shares) < k:
+                    raise ValueError(
+                        f"Failed to download enough chunks for original chunk {orig_idx}. "
+                        f"Need {k}, but only downloaded {len(downloaded_shares)}."
+                    )
+
+                # Reconstruct this chunk
+                decoder = zfec.Decoder(k, m)
+                reconstructed_data = decoder.decode(downloaded_shares, share_indexes)
+                
+                # If we used the sub-block approach during encoding, we need to recombine the sub-blocks
+                if isinstance(reconstructed_data, list):
+                    # Combine the sub-blocks back into a single chunk
+                    reconstructed_chunk = b''.join(reconstructed_data)
+                else:
+                    # The simple case where we didn't use sub-blocks
+                    reconstructed_chunk = reconstructed_data
+                    
+                reconstructed_chunks.append(reconstructed_chunk)
+
+                if verbose and (orig_idx + 1) % 10 == 0:
+                    print(
+                        f"  Reconstructed {orig_idx + 1}/{len(chunks_by_original)} chunks"
+                    )
+
+            # Step 5: Combine the reconstructed chunks into a file
+            if verbose:
+                print(f"Combining reconstructed chunks...")
+
+            # Concatenate all chunks
+            file_data = b"".join(reconstructed_chunks)
+
+            # Remove padding from the last chunk
+            if original_file["size"] < len(file_data):
+                file_data = file_data[: original_file["size"]]
+
+            # Step 6: Decrypt if necessary
+            if is_encrypted:
+                if not self.encryption_available:
+                    raise ValueError(
+                        "File is encrypted but encryption is not available. "
+                        "Install PyNaCl and configure an encryption key."
+                    )
+
+                if verbose:
+                    print(f"Decrypting file data...")
+
+                file_data = self.decrypt_data(file_data)
+
+            # Step 7: Write to the output file
+            with open(output_file, "wb") as f:
+                f.write(file_data)
+
+            # Step 8: Verify hash if available
+            if "hash" in original_file:
+                actual_hash = hashlib.sha256(file_data).hexdigest()
+                expected_hash = original_file["hash"]
+
+                if actual_hash != expected_hash:
+                    print(f"Warning: File hash mismatch!")
+                    print(f"  Expected: {expected_hash}")
+                    print(f"  Actual:   {actual_hash}")
+
+            if verbose:
+                print(f"Reconstruction complete!")
+                print(f"File saved to: {output_file}")
+
+            return output_file
+
+        finally:
+            # Clean up temporary directory if we created it
+            if temp_dir_obj is not None:
+                temp_dir_obj.close()
+
+    def store_erasure_coded_file(
+        self,
+        file_path: str,
+        k: int = 3,
+        m: int = 5,
+        chunk_size: int = 1024 * 1024,  # 1MB chunks
+        encrypt: Optional[bool] = None,
+        miner_ids: List[str] = None,
+        substrate_client=None,
+        max_retries: int = 3,
+        verbose: bool = True,
+    ) -> Dict[str, Any]:
+        """
+        Erasure code a file, upload the chunks to IPFS, and store in the Hippius marketplace.
+
+        This is a convenience method that combines erasure_code_file with storage_request.
+
+        Args:
+            file_path: Path to the file to upload
+            k: Number of data chunks (minimum required to reconstruct)
+            m: Total number of chunks (k + redundancy)
+            chunk_size: Size of each chunk in bytes before encoding
+            encrypt: Whether to encrypt the file before encoding
+            miner_ids: List of specific miner IDs to use for storage
+            substrate_client: SubstrateClient to use (or None to create one)
+            max_retries: Maximum number of retry attempts
+            verbose: Whether to print progress information
+
+        Returns:
+            dict: Result including metadata CID and transaction hash
+
+        Raises:
+            ValueError: If parameters are invalid
+            RuntimeError: If processing fails
+        """
+        # Step 1: Erasure code the file and upload chunks
+        metadata = self.erasure_code_file(
+            file_path=file_path,
+            k=k,
+            m=m,
+            chunk_size=chunk_size,
+            encrypt=encrypt,
+            max_retries=max_retries,
+            verbose=verbose,
+        )
+
+        # Step 2: Import substrate client if we need it
+        if substrate_client is None:
+            from hippius_sdk.substrate import SubstrateClient, FileInput
+
+            substrate_client = SubstrateClient()
+        else:
+            # Just get the FileInput class
+            from hippius_sdk.substrate import FileInput
+
+        original_file = metadata["original_file"]
+        metadata_cid = metadata["metadata_cid"]
+        
+        # Create a list to hold all the file inputs (metadata + all chunks)
+        all_file_inputs = []
+
+        # Step 3: Prepare metadata file for storage
+        if verbose:
+            print(f"Preparing to store metadata and {len(metadata['chunks'])} chunks in the Hippius marketplace...")
+
+        # Create a file input for the metadata file
+        metadata_file_input = FileInput(
+            file_hash=metadata_cid, file_name=f"{original_file['name']}.ec_metadata"
+        )
+        all_file_inputs.append(metadata_file_input)
+        
+        # Step 4: Add all chunks to the storage request
+        if verbose:
+            print(f"Adding all chunks to storage request...")
+            
+        for i, chunk in enumerate(metadata["chunks"]):
+            # Extract the CID string from the chunk's cid dictionary
+            chunk_cid = chunk["cid"]["cid"] if isinstance(chunk["cid"], dict) and "cid" in chunk["cid"] else chunk["cid"]
+            chunk_file_input = FileInput(
+                file_hash=chunk_cid,
+                file_name=chunk["name"]
+            )
+            all_file_inputs.append(chunk_file_input)
+            
+            # Print progress for large numbers of chunks
+            if verbose and (i + 1) % 50 == 0:
+                print(f"  Prepared {i + 1}/{len(metadata['chunks'])} chunks for storage")
+
+        # Step 5: Submit the storage request for all files
+        try:
+            if verbose:
+                print(f"Submitting storage request for 1 metadata file and {len(metadata['chunks'])} chunks...")
+                
+            tx_hash = substrate_client.storage_request(
+                files=all_file_inputs, miner_ids=miner_ids
+            )
+
+            if verbose:
+                print(f"Successfully stored all files in marketplace!")
+                print(f"Transaction hash: {tx_hash}")
+                print(f"Metadata CID: {metadata_cid}")
+                print(f"Total files stored: {len(all_file_inputs)} (1 metadata + {len(metadata['chunks'])} chunks)")
+
+            return {
+                "metadata": metadata,
+                "metadata_cid": metadata_cid,
+                "transaction_hash": tx_hash,
+                "total_files_stored": len(all_file_inputs)
+            }
+
+        except Exception as e:
+            print(f"Error storing files in marketplace: {str(e)}")
+            # Return the metadata even if storage fails
+            return {"metadata": metadata, "metadata_cid": metadata_cid, "error": str(e)}

hippius-0.1.0.dist-info/RECORD

@@ -1,9 +0,0 @@
-hippius_sdk/__init__.py,sha256=SwOREu9EJZ9ZRM-rSPX0o1hhsOUIADuP3CxoF4Mp_qI,288
-hippius_sdk/cli.py,sha256=ctg-dfe3uoXBx6McPenZmWE-5AZTLZ39Pro3xMRbAD8,22274
-hippius_sdk/client.py,sha256=Etj6u4Q0Y5KN4QxixOc8uy-zSuIsixx4TGLHXqGiHno,8888
-hippius_sdk/ipfs.py,sha256=IcPtC99I9CmBA3-sSfbnc0RMZ3d3Z0CRtCmRmf1hzR0,37905
-hippius_sdk/substrate.py,sha256=mfDxbKn9HdtcK1xEnj_BnnreRw8ITZswtDoBhtliidM,27278
-hippius-0.1.0.dist-info/METADATA,sha256=RHf-CbtSTQLKeIsfMnOjzOnpWlkmWKszd8JeoYwUCMM,13047
-hippius-0.1.0.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
-hippius-0.1.0.dist-info/entry_points.txt,sha256=b1lo60zRXmv1ud-c5BC-cJcAfGE5FD4qM_nia6XeQtM,98
-hippius-0.1.0.dist-info/RECORD,,