hippius 0.1.7__tar.gz → 0.1.9__tar.gz

{hippius-0.1.7 → hippius-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hippius
-Version: 0.1.7
+Version: 0.1.9
 Summary: Python SDK and CLI for Hippius blockchain storage
 Home-page: https://github.com/thenervelab/hippius-sdk
 Author: Dubs
@@ -67,7 +67,7 @@ client = HippiusClient()
 # Or specify custom endpoints
 client = HippiusClient(
     ipfs_gateway="https://ipfs.io",                       # For downloads (default)
-    ipfs_api_url="https://relay-fr.hippius.network",  # For uploads (default)
+    ipfs_api_url="https://store.hippius.network",  # For uploads (default)
 )
 
 # Upload a file to IPFS
@@ -313,6 +313,64 @@ This command provides detailed information about each erasure-coded file includi
 - Miners storing the file
 - Reconstruction command
 
+The `ec-files` command is optimized for performance through parallel processing and intelligent filtering, making it efficient even with large numbers of files.
+
+#### Performance Considerations
+
+The erasure coding implementation has been optimized for:
+
+1. **Speed**: Parallel processing for file operations
+2. **Memory efficiency**: Files are processed in chunks to minimize memory usage
+3. **Auto-tuning**: Parameters like chunk size are automatically adjusted for small files
+4. **Intelligent filtering**: The system can quickly identify potential erasure-coded files
+
+For extremely large files (>1GB), consider using larger chunk sizes to improve performance:
+
+```bash
+hippius erasure-code large_video.mp4 --chunk-size 10485760  # 10MB chunks
+```
+
+#### Directory Support for Erasure Coding
+
+Hippius SDK now supports applying erasure coding to entire directories:
+
+```bash
+# Apply erasure coding to an entire directory
+hippius erasure-code my_directory/
+
+# The CLI will detect that it's a directory and offer two options:
+# 1. Archive the directory first, then erasure code the archive
+# 2. Apply erasure coding to each file in the directory individually
+```
+
+When choosing the second option, the system will process each file in the directory individually, adjusting parameters like chunk size automatically for small files. A summary of the operation will be displayed, showing:
+- Total files processed
+- Successfully coded files with their metadata CIDs
+- Any files that failed, with error details
+
+To reconstruct a file from erasure-coded chunks:
+
+```bash
+hippius reconstruct QmMetadataCID reconstructed_filename
+```
+
+#### Default Address Management
+
+Hippius SDK now allows setting a default address for read-only operations, making it easier to use commands like `files` and `ec-files` without specifying an account address each time:
+
+```bash
+# Set a default address for read-only operations
+hippius address set-default 5H1QBRF7T7dgKwzVGCgS4wioudvMRf9K4NEDzfuKLnuyBNzH
+
+# View the currently set default address
+hippius address get-default
+
+# Clear the default address
+hippius address clear-default
+```
+
+Once a default address is set, commands like `hippius files` and `hippius ec-files` will automatically use this address when no explicit address is provided.
+
 #### Troubleshooting
 
 1. **IPFS Connection Issues**: Make sure you have either:
@@ -333,7 +391,30 @@ This command provides detailed information about each erasure-coded file includi
    which hippius
    ```
 
-4. **Substrate Issues**: For marketplace operations, make sure your `.env` has the correct `SUBSTRATE_SEED_PHRASE` and `SUBSTRATE_URL` values.
+4. **Default Address Issues**: If you receive errors about missing account address:
+   ```bash
+   # Set a default address for read-only operations
+   hippius address set-default 5H1QBRF7T7dgKwzVGCgS4wioudvMRf9K4NEDzfuKLnuyBNzH
+   ```
+
+5. **Substrate Issues**: For marketplace operations, make sure your `.env` has the correct `SUBSTRATE_SEED_PHRASE` and `SUBSTRATE_URL` values.
+
+6. **Erasure Coding Problems**:
+   - **"Wrong length for input blocks"**: This typically happens with very small files
+     ```bash
+     # Try smaller k and m values for small files
+     hippius erasure-code small_file.txt --k 2 --m 3
+     ```
+   - **Directories can't be directly coded**: Use the directory support option when prompted
+   - **"zfec is required"**: Install the missing package
+     ```bash
+     pip install zfec
+     poetry add zfec
+     ```
+   - **Slow performance with large files**: Increase chunk size
+     ```bash
+     hippius erasure-code large_file.mp4 --chunk-size 5242880  # 5MB chunks
+     ```
 
 ## Command Line Interface
 
@@ -346,7 +427,7 @@ The Hippius SDK includes a powerful command-line interface (CLI) that provides a
 hippius --help
 
 # Set global options
-hippius --gateway https://ipfs.io --api-url https://relay-fr.hippius.network --verbose
+hippius --gateway https://ipfs.io --api-url https://store.hippius.network --verbose
 ```
 
 ### IPFS Operations
@@ -431,7 +512,7 @@ hippius ec-files --all-miners
 hippius ec-files --show-chunks
 ```
 
-The `ec-files` command makes it easy to track and manage your erasure-coded files separately from regular files. It provides the metadata CID needed for reconstruction and information about chunk distribution.
+The `ec-files` command has been optimized for performance and can now handle large numbers of files efficiently through parallel processing.
 
 ### Using Environment Variables
 
@@ -439,7 +520,7 @@ The CLI automatically reads from your `.env` file for common settings:
 
 ```
 IPFS_GATEWAY=https://ipfs.io
-IPFS_API_URL=https://relay-fr.hippius.network
+IPFS_API_URL=https://store.hippius.network
 SUBSTRATE_URL=wss://rpc.hippius.network
 SUBSTRATE_SEED_PHRASE="your twelve word seed phrase..."
 SUBSTRATE_DEFAULT_MINERS=miner1,miner2,miner3
@@ -546,13 +627,14 @@ The configuration is organized in the following sections:
 {
   "ipfs": {
     "gateway": "https://ipfs.io",
-    "api_url": "https://relay-fr.hippius.network",
+    "api_url": "https://store.hippius.network",
     "local_ipfs": false
   },
   "substrate": {
     "url": "wss://rpc.hippius.network",
     "seed_phrase": null,
-    "default_miners": []
+    "default_miners": [],
+    "default_address": null
   },
   "encryption": {
     "encrypt_by_default": false,
@@ -737,7 +819,13 @@ hippius --help
    which hippius
    ```
 
-4. **Substrate Issues**: For marketplace operations, make sure your `.env` has the correct `SUBSTRATE_SEED_PHRASE` and `SUBSTRATE_URL` values.
+4. **Default Address Issues**: If you receive errors about missing account address:
+   ```bash
+   # Set a default address for read-only operations
+   hippius address set-default 5H1QBRF7T7dgKwzVGCgS4wioudvMRf9K4NEDzfuKLnuyBNzH
+   ```
+
+5. **Substrate Issues**: For marketplace operations, make sure your `.env` has the correct `SUBSTRATE_SEED_PHRASE` and `SUBSTRATE_URL` values.
 
 ## Contributing
 

{hippius-0.1.7 → hippius-0.1.9}/README.md

@@ -36,7 +36,7 @@ client = HippiusClient()
 # Or specify custom endpoints
 client = HippiusClient(
     ipfs_gateway="https://ipfs.io",                       # For downloads (default)
-    ipfs_api_url="https://relay-fr.hippius.network",  # For uploads (default)
+    ipfs_api_url="https://store.hippius.network",  # For uploads (default)
 )
 
 # Upload a file to IPFS
@@ -282,6 +282,64 @@ This command provides detailed information about each erasure-coded file includi
 - Miners storing the file
 - Reconstruction command
 
+The `ec-files` command is optimized for performance through parallel processing and intelligent filtering, making it efficient even with large numbers of files.
+
+#### Performance Considerations
+
+The erasure coding implementation has been optimized for:
+
+1. **Speed**: Parallel processing for file operations
+2. **Memory efficiency**: Files are processed in chunks to minimize memory usage
+3. **Auto-tuning**: Parameters like chunk size are automatically adjusted for small files
+4. **Intelligent filtering**: The system can quickly identify potential erasure-coded files
+
+For extremely large files (>1GB), consider using larger chunk sizes to improve performance:
+
+```bash
+hippius erasure-code large_video.mp4 --chunk-size 10485760  # 10MB chunks
+```
+
+#### Directory Support for Erasure Coding
+
+Hippius SDK now supports applying erasure coding to entire directories:
+
+```bash
+# Apply erasure coding to an entire directory
+hippius erasure-code my_directory/
+
+# The CLI will detect that it's a directory and offer two options:
+# 1. Archive the directory first, then erasure code the archive
+# 2. Apply erasure coding to each file in the directory individually
+```
+
+When choosing the second option, the system will process each file in the directory individually, adjusting parameters like chunk size automatically for small files. A summary of the operation will be displayed, showing:
+- Total files processed
+- Successfully coded files with their metadata CIDs
+- Any files that failed, with error details
+
+To reconstruct a file from erasure-coded chunks:
+
+```bash
+hippius reconstruct QmMetadataCID reconstructed_filename
+```
+
+#### Default Address Management
+
+Hippius SDK now allows setting a default address for read-only operations, making it easier to use commands like `files` and `ec-files` without specifying an account address each time:
+
+```bash
+# Set a default address for read-only operations
+hippius address set-default 5H1QBRF7T7dgKwzVGCgS4wioudvMRf9K4NEDzfuKLnuyBNzH
+
+# View the currently set default address
+hippius address get-default
+
+# Clear the default address
+hippius address clear-default
+```
+
+Once a default address is set, commands like `hippius files` and `hippius ec-files` will automatically use this address when no explicit address is provided.
+
 #### Troubleshooting
 
 1. **IPFS Connection Issues**: Make sure you have either:
@@ -302,7 +360,30 @@ This command provides detailed information about each erasure-coded file includi
    which hippius
    ```
 
-4. **Substrate Issues**: For marketplace operations, make sure your `.env` has the correct `SUBSTRATE_SEED_PHRASE` and `SUBSTRATE_URL` values.
+4. **Default Address Issues**: If you receive errors about missing account address:
+   ```bash
+   # Set a default address for read-only operations
+   hippius address set-default 5H1QBRF7T7dgKwzVGCgS4wioudvMRf9K4NEDzfuKLnuyBNzH
+   ```
+
+5. **Substrate Issues**: For marketplace operations, make sure your `.env` has the correct `SUBSTRATE_SEED_PHRASE` and `SUBSTRATE_URL` values.
+
+6. **Erasure Coding Problems**:
+   - **"Wrong length for input blocks"**: This typically happens with very small files
+     ```bash
+     # Try smaller k and m values for small files
+     hippius erasure-code small_file.txt --k 2 --m 3
+     ```
+   - **Directories can't be directly coded**: Use the directory support option when prompted
+   - **"zfec is required"**: Install the missing package
+     ```bash
+     pip install zfec
+     poetry add zfec
+     ```
+   - **Slow performance with large files**: Increase chunk size
+     ```bash
+     hippius erasure-code large_file.mp4 --chunk-size 5242880  # 5MB chunks
+     ```
 
 ## Command Line Interface
 
@@ -315,7 +396,7 @@ The Hippius SDK includes a powerful command-line interface (CLI) that provides a
 hippius --help
 
 # Set global options
-hippius --gateway https://ipfs.io --api-url https://relay-fr.hippius.network --verbose
+hippius --gateway https://ipfs.io --api-url https://store.hippius.network --verbose
 ```
 
 ### IPFS Operations
@@ -400,7 +481,7 @@ hippius ec-files --all-miners
 hippius ec-files --show-chunks
 ```
 
-The `ec-files` command makes it easy to track and manage your erasure-coded files separately from regular files. It provides the metadata CID needed for reconstruction and information about chunk distribution.
+The `ec-files` command has been optimized for performance and can now handle large numbers of files efficiently through parallel processing.
 
 ### Using Environment Variables
 
@@ -408,7 +489,7 @@ The CLI automatically reads from your `.env` file for common settings:
 
 ```
 IPFS_GATEWAY=https://ipfs.io
-IPFS_API_URL=https://relay-fr.hippius.network
+IPFS_API_URL=https://store.hippius.network
 SUBSTRATE_URL=wss://rpc.hippius.network
 SUBSTRATE_SEED_PHRASE="your twelve word seed phrase..."
 SUBSTRATE_DEFAULT_MINERS=miner1,miner2,miner3
@@ -515,13 +596,14 @@ The configuration is organized in the following sections:
 {
   "ipfs": {
     "gateway": "https://ipfs.io",
-    "api_url": "https://relay-fr.hippius.network",
+    "api_url": "https://store.hippius.network",
     "local_ipfs": false
   },
   "substrate": {
     "url": "wss://rpc.hippius.network",
     "seed_phrase": null,
-    "default_miners": []
+    "default_miners": [],
+    "default_address": null
   },
   "encryption": {
     "encrypt_by_default": false,
@@ -706,7 +788,13 @@ hippius --help
    which hippius
    ```
 
-4. **Substrate Issues**: For marketplace operations, make sure your `.env` has the correct `SUBSTRATE_SEED_PHRASE` and `SUBSTRATE_URL` values.
+4. **Default Address Issues**: If you receive errors about missing account address:
+   ```bash
+   # Set a default address for read-only operations
+   hippius address set-default 5H1QBRF7T7dgKwzVGCgS4wioudvMRf9K4NEDzfuKLnuyBNzH
+   ```
+
+5. **Substrate Issues**: For marketplace operations, make sure your `.env` has the correct `SUBSTRATE_SEED_PHRASE` and `SUBSTRATE_URL` values.
 
 ## Contributing
 

{hippius-0.1.7 → hippius-0.1.9}/hippius_sdk/cli.py

@@ -1608,8 +1608,8 @@ examples:
     )
     parser.add_argument(
         "--api-url",
-        default=get_config_value("ipfs", "api_url", "https://relay-fr.hippius.network"),
-        help="IPFS API URL for uploads (default: from config or https://relay-fr.hippius.network)",
+        default=get_config_value("ipfs", "api_url", "https://store.hippius.network"),
+        help="IPFS API URL for uploads (default: from config or https://store.hippius.network)",
     )
     parser.add_argument(
         "--local-ipfs",

{hippius-0.1.7 → hippius-0.1.9}/hippius_sdk/client.py

@@ -49,7 +49,7 @@ class HippiusClient:
 
         if ipfs_api_url is None:
             ipfs_api_url = get_config_value(
-                "ipfs", "api_url", "https://relay-fr.hippius.network"
+                "ipfs", "api_url", "https://store.hippius.network"
             )
 
             # Check if local IPFS is enabled in config

{hippius-0.1.7 → hippius-0.1.9}/hippius_sdk/config.py

@@ -19,7 +19,7 @@ CONFIG_FILE = os.path.join(CONFIG_DIR, "config.json")
 DEFAULT_CONFIG = {
     "ipfs": {
         "gateway": "https://ipfs.io",
-        "api_url": "https://relay-fr.hippius.network",
+        "api_url": "https://store.hippius.network",
         "local_ipfs": False,
     },
     "substrate": {

{hippius-0.1.7 → hippius-0.1.9}/hippius_sdk/ipfs.py

@@ -62,7 +62,7 @@ class IPFSClient:
 
         if api_url is None:
             api_url = get_config_value(
-                "ipfs", "api_url", "https://relay-fr.hippius.network"
+                "ipfs", "api_url", "https://store.hippius.network"
             )
 
             # Check if local IPFS is enabled in config

{hippius-0.1.7 → hippius-0.1.9}/hippius_sdk/substrate.py

@@ -6,6 +6,7 @@ Note: This functionality is coming soon and not implemented yet.
 
 import os
 import json
+import uuid
 from typing import Dict, Any, Optional, List, Union
 from substrateinterface import SubstrateInterface, Keypair
 from dotenv import load_dotenv
@@ -205,8 +206,8 @@ class SubstrateClient:
         """
         Submit a storage request for IPFS files to the marketplace.
 
-        This method batches all files into a single transaction to efficiently store
-        multiple files at once.
+        This method creates a JSON file with the list of files to pin, uploads it to IPFS,
+        and submits the CID of this file to the chain.
 
         Args:
             files: List of FileInput objects or dictionaries with fileHash and fileName
@@ -241,7 +242,7 @@ class SubstrateClient:
                 file_inputs.append(file)
 
         # Print what is being submitted
-        print(f"Submitting storage request for {len(file_inputs)} files as a batch:")
+        print(f"Preparing storage request for {len(file_inputs)} files:")
         for file in file_inputs:
             print(f"  - {file.file_name}: {file.file_hash}")
 
@@ -261,22 +262,50 @@ class SubstrateClient:
                 )
                 print(f"Connected to Substrate node at {self.url}")
 
-            # Format files for the batch call - all files are included in a single array
-            formatted_files = []
+            # Step 1: Create a JSON file with the list of files to pin
+            file_list = []
             for file_input in file_inputs:
-                formatted_files.append(
-                    {
-                        "file_hash": file_input.file_hash,
-                        "file_name": file_input.file_name,
-                    }
+                file_list.append(
+                    {"filename": file_input.file_name, "cid": file_input.file_hash}
                 )
 
-            # Create call parameters with all files in a single batch
+            # Convert to JSON
+            files_json = json.dumps(file_list, indent=2)
+            print(f"Created file list with {len(file_list)} entries")
+
+            # Step 2: Upload the JSON file to IPFS
+            import tempfile
+            from hippius_sdk.ipfs import IPFSClient
+
+            ipfs_client = IPFSClient()
+
+            # Create a temporary file with the JSON content
+            with tempfile.NamedTemporaryFile(
+                mode="w+", suffix=".json", delete=False
+            ) as temp_file:
+                temp_file_path = temp_file.name
+                temp_file.write(files_json)
+
+            try:
+                print("Uploading file list to IPFS...")
+                upload_result = ipfs_client.upload_file(temp_file_path)
+                files_list_cid = upload_result["cid"]
+                print(f"File list uploaded to IPFS with CID: {files_list_cid}")
+            finally:
+                # Clean up the temporary file
+                if os.path.exists(temp_file_path):
+                    os.remove(temp_file_path)
+
+            # Step 3: Submit the CID of the JSON file to the chain
+            # Create call parameters with the CID of the JSON file
             call_params = {
-                "files_input": formatted_files,
-                "miner_ids": miner_ids
-                if miner_ids
-                else [],  # Always include miner_ids, empty array if not specified
+                "files_input": [
+                    {
+                        "file_hash": files_list_cid,
+                        "file_name": f"files_list_{uuid.uuid4()}",  # Generate a unique ID
+                    }
+                ],
+                "miner_ids": miner_ids if miner_ids else [],
             }
 
             # Create the call to the marketplace
@@ -300,7 +329,9 @@ class SubstrateClient:
                 call=call, keypair=self._keypair
             )
 
-            print(f"Submitting batch transaction for {len(formatted_files)} files...")
+            print(
+                f"Submitting transaction to store {len(file_list)} files via file list CID..."
+            )
 
             # Submit the transaction
             response = self._substrate.submit_extrinsic(
@@ -310,11 +341,10 @@ class SubstrateClient:
             # Get the transaction hash
             tx_hash = response.extrinsic_hash
 
-            print(f"Batch transaction submitted successfully!")
+            print(f"Transaction submitted successfully!")
             print(f"Transaction hash: {tx_hash}")
-            print(
-                f"All {len(formatted_files)} files have been stored in a single transaction"
-            )
+            print(f"File list CID: {files_list_cid}")
+            print(f"All {len(file_list)} files will be stored through this request")
 
             return tx_hash
 

{hippius-0.1.7 → hippius-0.1.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "hippius"
-version = "0.1.7"
+version = "0.1.9"
 description = "Python SDK and CLI for Hippius blockchain storage"
 authors = ["Dubs <dubs@dubs.rs>"]
 readme = "README.md"