hippius 0.1.6__py3-none-any.whl → 0.1.9__py3-none-any.whl

hippius_sdk/ipfs.py

@@ -13,6 +13,10 @@ import uuid
 from typing import Dict, Any, Optional, Union, List, Tuple
 import ipfshttpclient
 from dotenv import load_dotenv
+from hippius_sdk.config import (
+    get_config_value,
+    get_encryption_key,
+)
 
 # Import PyNaCl for encryption
 try:
@@ -37,8 +41,8 @@ class IPFSClient:
 
     def __init__(
         self,
-        gateway: str = "https://ipfs.io",
-        api_url: Optional[str] = "https://relay-fr.hippius.network",
+        gateway: Optional[str] = None,
+        api_url: Optional[str] = None,
         encrypt_by_default: Optional[bool] = None,
         encryption_key: Optional[bytes] = None,
     ):
@@ -46,12 +50,25 @@ class IPFSClient:
         Initialize the IPFS client.
 
         Args:
-            gateway: IPFS gateway URL for downloading content
-            api_url: IPFS API URL for uploading content. Defaults to Hippius relay node.
+            gateway: IPFS gateway URL for downloading content (from config if None)
+            api_url: IPFS API URL for uploading content (from config if None)
                     Set to None to try to connect to a local IPFS daemon.
-            encrypt_by_default: Whether to encrypt files by default (from .env if None)
-            encryption_key: Encryption key for NaCl secretbox (from .env if None)
+            encrypt_by_default: Whether to encrypt files by default (from config if None)
+            encryption_key: Encryption key for NaCl secretbox (from config if None)
         """
+        # Load configuration values if not explicitly provided
+        if gateway is None:
+            gateway = get_config_value("ipfs", "gateway", "https://ipfs.io")
+
+        if api_url is None:
+            api_url = get_config_value(
+                "ipfs", "api_url", "https://store.hippius.network"
+            )
+
+            # Check if local IPFS is enabled in config
+            if get_config_value("ipfs", "local_ipfs", False):
+                api_url = "http://localhost:5001"
+
         self.gateway = gateway.rstrip("/")
         self.api_url = api_url
         self.client = None
@@ -92,7 +109,7 @@ class IPFSClient:
     def _initialize_encryption(
         self, encrypt_by_default: Optional[bool], encryption_key: Optional[bytes]
     ):
-        """Initialize encryption settings from parameters or .env file."""
+        """Initialize encryption settings from parameters or configuration."""
         # Check if encryption is available
         if not ENCRYPTION_AVAILABLE:
             self.encryption_available = False
@@ -100,31 +117,19 @@ class IPFSClient:
             self.encryption_key = None
             return
 
-        # Load environment variables
-        load_dotenv()
-
-        # Set up encryption default from parameter or .env
-        self.encrypt_by_default = encrypt_by_default
-        if self.encrypt_by_default is None:
-            env_default = os.getenv("HIPPIUS_ENCRYPT_BY_DEFAULT", "false").lower()
-            self.encrypt_by_default = env_default in ("true", "1", "yes")
+        # Set up encryption default from parameter or config
+        if encrypt_by_default is None:
+            self.encrypt_by_default = get_config_value(
+                "encryption", "encrypt_by_default", False
+            )
+        else:
+            self.encrypt_by_default = encrypt_by_default
 
-        # Set up encryption key from parameter or .env
-        self.encryption_key = encryption_key
-        if self.encryption_key is None:
-            env_key = os.getenv("HIPPIUS_ENCRYPTION_KEY")
-            if env_key:
-                try:
-                    self.encryption_key = base64.b64decode(env_key)
-                    # Validate key length
-                    if len(self.encryption_key) != nacl.secret.SecretBox.KEY_SIZE:
-                        print(
-                            f"Warning: Encryption key from .env has incorrect length. Expected {nacl.secret.SecretBox.KEY_SIZE} bytes, got {len(self.encryption_key)} bytes."
-                        )
-                        self.encryption_key = None
-                except Exception as e:
-                    print(f"Warning: Failed to decode encryption key from .env: {e}")
-                    self.encryption_key = None
+        # Set up encryption key from parameter or config
+        if encryption_key is None:
+            self.encryption_key = get_encryption_key()
+        else:
+            self.encryption_key = encryption_key
 
         # Check if we have a valid key and can encrypt
         self.encryption_available = (
@@ -726,7 +731,11 @@ class IPFSClient:
         return cid
 
     def download_file(
-        self, cid: str, output_path: str, decrypt: Optional[bool] = None
+        self,
+        cid: str,
+        output_path: str,
+        decrypt: Optional[bool] = None,
+        max_retries: int = 3,
     ) -> Dict[str, Any]:
         """
         Download a file from IPFS with optional decryption.
@@ -735,6 +744,7 @@ class IPFSClient:
             cid: Content Identifier (CID) of the file to download
             output_path: Path where the downloaded file will be saved
             decrypt: Whether to decrypt the file (overrides default)
+            max_retries: Maximum number of retry attempts (default: 3)
 
         Returns:
             Dict[str, Any]: Dictionary containing download results:
@@ -772,16 +782,41 @@ class IPFSClient:
             else:
                 download_path = output_path
 
-            # Download the file
-            url = f"{self.gateway}/ipfs/{cid}"
-            response = requests.get(url, stream=True)
-            response.raise_for_status()
+            # Download the file with retry logic
+            retries = 0
+            last_error = None
+
+            while retries < max_retries:
+                try:
+                    # Download the file
+                    url = f"{self.gateway}/ipfs/{cid}"
+                    response = requests.get(url, stream=True)
+                    response.raise_for_status()
+
+                    os.makedirs(
+                        os.path.dirname(os.path.abspath(download_path)), exist_ok=True
+                    )
+
+                    with open(download_path, "wb") as f:
+                        for chunk in response.iter_content(chunk_size=8192):
+                            f.write(chunk)
 
-            os.makedirs(os.path.dirname(os.path.abspath(download_path)), exist_ok=True)
+                    # If we reach here, download was successful
+                    break
 
-            with open(download_path, "wb") as f:
-                for chunk in response.iter_content(chunk_size=8192):
-                    f.write(chunk)
+                except (requests.exceptions.RequestException, IOError) as e:
+                    # Save the error and retry
+                    last_error = e
+                    retries += 1
+
+                    if retries < max_retries:
+                        wait_time = 2**retries  # Exponential backoff: 2, 4, 8 seconds
+                        print(f"Download attempt {retries} failed: {str(e)}")
+                        print(f"Retrying in {wait_time} seconds...")
+                        time.sleep(wait_time)
+                    else:
+                        # Raise the last error if we've exhausted all retries
+                        raise
 
             # Decrypt if needed
             if should_decrypt:
@@ -1096,39 +1131,41 @@ class IPFSClient:
                 print(
                     f"Warning: File has fewer chunks ({len(chunks)}) than k={k}. Adjusting parameters."
                 )
-            
+
             # If we have a very small file, we'll just use a single chunk
             # but will still split it into k sub-blocks during encoding
             if len(chunks) == 1:
                 if verbose:
-                    print(f"Small file (single chunk): will split into {k} sub-blocks for encoding")
+                    print(
+                        f"Small file (single chunk): will split into {k} sub-blocks for encoding"
+                    )
             else:
                 # If we have multiple chunks but fewer than k, adjust k to match
                 old_k = k
                 k = max(1, len(chunks))
                 if verbose:
                     print(f"Adjusting k from {old_k} to {k} to match available chunks")
-                    
+
             # Ensure m is greater than k for redundancy
             if m <= k:
                 old_m = m
                 m = k + 2  # Ensure we have at least 2 redundant chunks
                 if verbose:
                     print(f"Adjusting m from {old_m} to {m} to ensure redundancy")
-            
+
             if verbose:
                 print(f"New parameters: k={k}, m={m}")
-                
+
         # Ensure we have at least one chunk to process
         if not chunks:
             raise ValueError("File is empty or too small to process")
-            
+
         # For k=1 case, ensure we have proper sized input for zfec
         if k == 1 and len(chunks) == 1:
             # zfec expects the input to be exactly chunk_size for k=1
             # So we need to pad if shorter or truncate if longer
             if len(chunks[0]) != chunk_size:
-                chunks[0] = chunks[0].ljust(chunk_size, b'\0')[:chunk_size]
+                chunks[0] = chunks[0].ljust(chunk_size, b"\0")[:chunk_size]
 
         # Create metadata
         metadata = {
@@ -1156,41 +1193,47 @@ class IPFSClient:
         for i, chunk in enumerate(chunks):
             try:
                 # For zfec encoder.encode(), we must provide exactly k blocks
-                
+
                 # Calculate how many bytes each sub-block should have
-                sub_block_size = (len(chunk) + k - 1) // k  # ceiling division for even distribution
-                
+                sub_block_size = (
+                    len(chunk) + k - 1
+                ) // k  # ceiling division for even distribution
+
                 # Split the chunk into exactly k sub-blocks of equal size (padding as needed)
                 sub_blocks = []
                 for j in range(k):
                     start = j * sub_block_size
                     end = min(start + sub_block_size, len(chunk))
                     sub_block = chunk[start:end]
-                    
+
                     # Pad if needed to make all sub-blocks the same size
                     if len(sub_block) < sub_block_size:
-                        sub_block = sub_block.ljust(sub_block_size, b'\0')
-                        
+                        sub_block = sub_block.ljust(sub_block_size, b"\0")
+
                     sub_blocks.append(sub_block)
-                
+
                 # Verify we have exactly k sub-blocks
                 if len(sub_blocks) != k:
-                    raise ValueError(f"Expected {k} sub-blocks but got {len(sub_blocks)}")
-                
+                    raise ValueError(
+                        f"Expected {k} sub-blocks but got {len(sub_blocks)}"
+                    )
+
                 # Encode the k sub-blocks to create m encoded blocks
                 encoder = zfec.Encoder(k, m)
                 encoded_chunks = encoder.encode(sub_blocks)
-                
+
                 # Add to our collection
                 all_encoded_chunks.append(encoded_chunks)
-                
+
                 if verbose and (i + 1) % 10 == 0:
                     print(f"  Encoded {i+1}/{len(chunks)} chunks")
             except Exception as e:
                 # If encoding fails, provide more helpful error message
                 error_msg = f"Error encoding chunk {i}: {str(e)}"
                 print(f"Error details: chunk size={len(chunk)}, k={k}, m={m}")
-                print(f"Sub-blocks created: {len(sub_blocks) if 'sub_blocks' in locals() else 'None'}")
+                print(
+                    f"Sub-blocks created: {len(sub_blocks) if 'sub_blocks' in locals() else 'None'}"
+                )
                 raise RuntimeError(f"{error_msg}")
 
         # Step 4: Upload all chunks to IPFS
@@ -1249,10 +1292,12 @@ class IPFSClient:
                 print(f"Uploading metadata file...")
 
             # Upload the metadata file to IPFS
-            metadata_cid_result = self.upload_file(metadata_path, max_retries=max_retries)
-            
+            metadata_cid_result = self.upload_file(
+                metadata_path, max_retries=max_retries
+            )
+
             # Extract just the CID string from the result dictionary
-            metadata_cid = metadata_cid_result['cid']
+            metadata_cid = metadata_cid_result["cid"]
             metadata["metadata_cid"] = metadata_cid
 
             if verbose:
@@ -1294,6 +1339,9 @@ class IPFSClient:
                 "Erasure coding is not available. Install zfec: pip install zfec"
             )
 
+        # Start timing the reconstruction process
+        start_time = time.time()
+
         # Create a temporary directory if not provided
         if temp_dir is None:
             temp_dir_obj = tempfile.TemporaryDirectory()
@@ -1309,6 +1357,10 @@ class IPFSClient:
             metadata_path = os.path.join(temp_dir, "metadata.json")
             self.download_file(metadata_cid, metadata_path, max_retries=max_retries)
 
+            if verbose:
+                metadata_download_time = time.time() - start_time
+                print(f"Metadata downloaded in {metadata_download_time:.2f} seconds")
+
             with open(metadata_path, "r") as f:
                 metadata = json.load(f)
 
@@ -1326,8 +1378,11 @@ class IPFSClient:
                 print(
                     f"File: {original_file['name']} ({original_file['size']/1024/1024:.2f} MB)"
                 )
-                print(f"Erasure coding parameters: k={k}, m={m}")
-                print(f"Encrypted: {is_encrypted}")
+                print(
+                    f"Erasure coding parameters: k={k}, m={m} (need {k} of {m} chunks to reconstruct)"
+                )
+                if is_encrypted:
+                    print(f"Encrypted: Yes")
 
             # Step 3: Group chunks by their original chunk index
             chunks_by_original = {}
@@ -1339,9 +1394,15 @@ class IPFSClient:
 
             # Step 4: For each original chunk, download at least k shares
             if verbose:
-                print(f"Downloading and reconstructing chunks...")
+                total_original_chunks = len(chunks_by_original)
+                total_chunks_to_download = total_original_chunks * k
+                print(
+                    f"Downloading and reconstructing {total_chunks_to_download} chunks..."
+                )
 
             reconstructed_chunks = []
+            chunks_downloaded = 0
+            chunks_failed = 0
 
             for orig_idx in sorted(chunks_by_original.keys()):
                 available_chunks = chunks_by_original[orig_idx]
@@ -1363,10 +1424,15 @@ class IPFSClient:
                     chunk_path = os.path.join(temp_dir, chunk["name"])
                     try:
                         # Extract the CID string from the chunk's cid dictionary
-                        chunk_cid = chunk["cid"]["cid"] if isinstance(chunk["cid"], dict) and "cid" in chunk["cid"] else chunk["cid"]
+                        chunk_cid = (
+                            chunk["cid"]["cid"]
+                            if isinstance(chunk["cid"], dict) and "cid" in chunk["cid"]
+                            else chunk["cid"]
+                        )
                         self.download_file(
                             chunk_cid, chunk_path, max_retries=max_retries
                         )
+                        chunks_downloaded += 1
 
                         # Read the chunk data
                         with open(chunk_path, "rb") as f:
@@ -1378,6 +1444,7 @@ class IPFSClient:
                     except Exception as e:
                         if verbose:
                             print(f"Error downloading chunk {chunk['name']}: {str(e)}")
+                        chunks_failed += 1
                         # Continue to the next chunk
 
                 # If we don't have enough chunks, try to download more
@@ -1390,20 +1457,32 @@ class IPFSClient:
                 # Reconstruct this chunk
                 decoder = zfec.Decoder(k, m)
                 reconstructed_data = decoder.decode(downloaded_shares, share_indexes)
-                
+
                 # If we used the sub-block approach during encoding, we need to recombine the sub-blocks
                 if isinstance(reconstructed_data, list):
                     # Combine the sub-blocks back into a single chunk
-                    reconstructed_chunk = b''.join(reconstructed_data)
+                    reconstructed_chunk = b"".join(reconstructed_data)
                 else:
                     # The simple case where we didn't use sub-blocks
                     reconstructed_chunk = reconstructed_data
-                    
+
                 reconstructed_chunks.append(reconstructed_chunk)
 
-                if verbose and (orig_idx + 1) % 10 == 0:
+                # Print progress
+                if verbose:
+                    progress_pct = (orig_idx + 1) / total_original_chunks * 100
                     print(
-                        f"  Reconstructed {orig_idx + 1}/{len(chunks_by_original)} chunks"
+                        f"  Progress: {orig_idx + 1}/{total_original_chunks} chunks ({progress_pct:.1f}%)"
+                    )
+
+            if verbose:
+                download_time = time.time() - start_time
+                print(
+                    f"Downloaded {chunks_downloaded} chunks in {download_time:.2f} seconds"
+                )
+                if chunks_failed > 0:
+                    print(
+                        f"Failed to download {chunks_failed} chunks (not needed for reconstruction)"
                     )
 
             # Step 5: Combine the reconstructed chunks into a file
@@ -1443,9 +1522,12 @@ class IPFSClient:
                     print(f"Warning: File hash mismatch!")
                     print(f"  Expected: {expected_hash}")
                     print(f"  Actual:   {actual_hash}")
+                elif verbose:
+                    print(f"Hash verification successful!")
 
+            total_time = time.time() - start_time
             if verbose:
-                print(f"Reconstruction complete!")
+                print(f"Reconstruction complete in {total_time:.2f} seconds!")
                 print(f"File saved to: {output_file}")
 
             return output_file
@@ -1453,7 +1535,7 @@ class IPFSClient:
         finally:
             # Clean up temporary directory if we created it
             if temp_dir_obj is not None:
-                temp_dir_obj.close()
+                temp_dir_obj.cleanup()
 
     def store_erasure_coded_file(
         self,
@@ -1512,42 +1594,49 @@ class IPFSClient:
 
         original_file = metadata["original_file"]
         metadata_cid = metadata["metadata_cid"]
-        
+
         # Create a list to hold all the file inputs (metadata + all chunks)
         all_file_inputs = []
 
         # Step 3: Prepare metadata file for storage
         if verbose:
-            print(f"Preparing to store metadata and {len(metadata['chunks'])} chunks in the Hippius marketplace...")
+            print(
+                f"Preparing to store metadata and {len(metadata['chunks'])} chunks in the Hippius marketplace..."
+            )
 
         # Create a file input for the metadata file
         metadata_file_input = FileInput(
             file_hash=metadata_cid, file_name=f"{original_file['name']}.ec_metadata"
         )
         all_file_inputs.append(metadata_file_input)
-        
+
         # Step 4: Add all chunks to the storage request
         if verbose:
             print(f"Adding all chunks to storage request...")
-            
+
         for i, chunk in enumerate(metadata["chunks"]):
             # Extract the CID string from the chunk's cid dictionary
-            chunk_cid = chunk["cid"]["cid"] if isinstance(chunk["cid"], dict) and "cid" in chunk["cid"] else chunk["cid"]
-            chunk_file_input = FileInput(
-                file_hash=chunk_cid,
-                file_name=chunk["name"]
+            chunk_cid = (
+                chunk["cid"]["cid"]
+                if isinstance(chunk["cid"], dict) and "cid" in chunk["cid"]
+                else chunk["cid"]
             )
+            chunk_file_input = FileInput(file_hash=chunk_cid, file_name=chunk["name"])
             all_file_inputs.append(chunk_file_input)
-            
+
             # Print progress for large numbers of chunks
             if verbose and (i + 1) % 50 == 0:
-                print(f"  Prepared {i + 1}/{len(metadata['chunks'])} chunks for storage")
+                print(
+                    f"  Prepared {i + 1}/{len(metadata['chunks'])} chunks for storage"
+                )
 
         # Step 5: Submit the storage request for all files
         try:
             if verbose:
-                print(f"Submitting storage request for 1 metadata file and {len(metadata['chunks'])} chunks...")
-                
+                print(
+                    f"Submitting storage request for 1 metadata file and {len(metadata['chunks'])} chunks..."
+                )
+
             tx_hash = substrate_client.storage_request(
                 files=all_file_inputs, miner_ids=miner_ids
             )
@@ -1556,13 +1645,15 @@ class IPFSClient:
                 print(f"Successfully stored all files in marketplace!")
                 print(f"Transaction hash: {tx_hash}")
                 print(f"Metadata CID: {metadata_cid}")
-                print(f"Total files stored: {len(all_file_inputs)} (1 metadata + {len(metadata['chunks'])} chunks)")
+                print(
+                    f"Total files stored: {len(all_file_inputs)} (1 metadata + {len(metadata['chunks'])} chunks)"
+                )
 
             return {
                 "metadata": metadata,
                 "metadata_cid": metadata_cid,
                 "transaction_hash": tx_hash,
-                "total_files_stored": len(all_file_inputs)
+                "total_files_stored": len(all_file_inputs),
             }
 
         except Exception as e: