hippius 0.2.3__py3-none-any.whl → 0.2.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
hippius_sdk/ipfs.py CHANGED
@@ -16,6 +16,13 @@ import httpx
16
16
  import requests
17
17
 
18
18
  from hippius_sdk.config import get_config_value, get_encryption_key
19
+ from hippius_sdk.errors import (
20
+ HippiusAlreadyDeletedError,
21
+ HippiusFailedIPFSUnpin,
22
+ HippiusFailedSubstrateDelete,
23
+ HippiusIPFSConnectionError,
24
+ HippiusMetadataError,
25
+ )
19
26
  from hippius_sdk.ipfs_core import AsyncIPFSClient
20
27
  from hippius_sdk.substrate import FileInput, SubstrateClient
21
28
  from hippius_sdk.utils import format_cid, format_size
@@ -439,10 +446,11 @@ class IPFSClient:
439
446
  ) -> Dict[str, Any]:
440
447
  """
441
448
  Download a file from IPFS with optional decryption.
449
+ Supports downloading directories - in that case, a directory structure will be created.
442
450
 
443
451
  Args:
444
452
  cid: Content Identifier (CID) of the file to download
445
- output_path: Path where the downloaded file will be saved
453
+ output_path: Path where the downloaded file/directory will be saved
446
454
  decrypt: Whether to decrypt the file (overrides default)
447
455
  max_retries: Maximum number of retry attempts (default: 3)
448
456
 
@@ -454,6 +462,7 @@ class IPFSClient:
454
462
  - size_formatted: Human-readable file size
455
463
  - elapsed_seconds: Time taken for the download in seconds
456
464
  - decrypted: Whether the file was decrypted
465
+ - is_directory: Whether the download was a directory
457
466
 
458
467
  Raises:
459
468
  requests.RequestException: If the download fails
@@ -461,6 +470,45 @@ class IPFSClient:
461
470
  """
462
471
  start_time = time.time()
463
472
 
473
+ # Use the improved ls function to properly detect directories
474
+ is_directory = False
475
+ try:
476
+ # The ls function now properly detects directories
477
+ ls_result = await self.client.ls(cid)
478
+ is_directory = ls_result.get("is_directory", False)
479
+ except Exception:
480
+ # If ls fails, we'll proceed as if it's a file
481
+ pass
482
+
483
+ # If it's a directory, handle it differently
484
+ if is_directory:
485
+ # For directories, we don't need to decrypt each file during the initial download
486
+ # We'll use the AsyncIPFSClient's download_directory method directly
487
+ try:
488
+ await self.client.download_directory(cid, output_path)
489
+
490
+ # Calculate the total size of the directory
491
+ total_size = 0
492
+ for root, _, files in os.walk(output_path):
493
+ for file in files:
494
+ file_path = os.path.join(root, file)
495
+ total_size += os.path.getsize(file_path)
496
+
497
+ elapsed_time = time.time() - start_time
498
+
499
+ return {
500
+ "success": True,
501
+ "output_path": output_path,
502
+ "size_bytes": total_size,
503
+ "size_formatted": self.format_size(total_size),
504
+ "elapsed_seconds": round(elapsed_time, 2),
505
+ "decrypted": False, # Directories aren't decrypted as a whole
506
+ "is_directory": True,
507
+ }
508
+ except Exception as e:
509
+ raise RuntimeError(f"Failed to download directory: {str(e)}")
510
+
511
+ # For regular files, use the existing logic
464
512
  # Determine if we should decrypt
465
513
  should_decrypt = self.encrypt_by_default if decrypt is None else decrypt
466
514
 
@@ -482,38 +530,11 @@ class IPFSClient:
482
530
  else:
483
531
  download_path = output_path
484
532
 
485
- # Download the file with retry logic
486
- retries = 0
487
- while retries < max_retries:
488
- try:
489
- # Download the file
490
- url = f"{self.gateway}/ipfs/{cid}"
491
- response = requests.get(url, stream=True)
492
- response.raise_for_status()
493
-
494
- os.makedirs(
495
- os.path.dirname(os.path.abspath(download_path)), exist_ok=True
496
- )
497
-
498
- with open(download_path, "wb") as f:
499
- for chunk in response.iter_content(chunk_size=8192):
500
- f.write(chunk)
501
-
502
- # If we reach here, download was successful
503
- break
533
+ await self.client.download_file(cid, download_path)
534
+ download_success = True
504
535
 
505
- except (requests.exceptions.RequestException, IOError) as e:
506
- # Save the error and retry
507
- retries += 1
508
-
509
- if retries < max_retries:
510
- wait_time = 2**retries # Exponential backoff: 2, 4, 8 seconds
511
- print(f"Download attempt {retries} failed: {str(e)}")
512
- print(f"Retrying in {wait_time} seconds...")
513
- time.sleep(wait_time)
514
- else:
515
- # Raise the last error if we've exhausted all retries
516
- raise
536
+ if not download_success:
537
+ raise RuntimeError("Failed to download file after multiple attempts")
517
538
 
518
539
  # Decrypt if needed
519
540
  if should_decrypt:
@@ -548,6 +569,7 @@ class IPFSClient:
548
569
  "size_formatted": self.format_size(file_size_bytes),
549
570
  "elapsed_seconds": round(elapsed_time, 2),
550
571
  "decrypted": should_decrypt,
572
+ "is_directory": False,
551
573
  }
552
574
 
553
575
  finally:
@@ -816,6 +838,9 @@ class IPFSClient:
816
838
  file_data = self.encrypt_data(file_data)
817
839
 
818
840
  # Step 2: Split the file into chunks for erasure coding
841
+ chunk_size = int(chunk_size)
842
+ chunk_size = max(1, chunk_size) # Ensure it's at least 1 byte
843
+
819
844
  chunks = []
820
845
  chunk_positions = []
821
846
  for i in range(0, len(file_data), chunk_size):
@@ -825,7 +850,7 @@ class IPFSClient:
825
850
 
826
851
  # Pad the last chunk if necessary
827
852
  if chunks and len(chunks[-1]) < chunk_size:
828
- pad_size = chunk_size - len(chunks[-1])
853
+ pad_size = int(chunk_size - len(chunks[-1]))
829
854
  chunks[-1] = chunks[-1] + b"\0" * pad_size
830
855
 
831
856
  # If we don't have enough chunks for the requested parameters, adjust
@@ -1028,11 +1053,20 @@ class IPFSClient:
1028
1053
 
1029
1054
  # Step 5: Create and upload the metadata file
1030
1055
  metadata_path = os.path.join(temp_dir, f"{file_id}_metadata.json")
1031
- with open(metadata_path, "w") as f:
1032
- json.dump(metadata, f, indent=2)
1056
+
1057
+ # Use binary mode to avoid any platform-specific text encoding issues
1058
+ with open(metadata_path, "wb") as f:
1059
+ # Encode the JSON with UTF-8 encoding explicitly
1060
+ metadata_json = json.dumps(metadata, indent=2, ensure_ascii=False)
1061
+ f.write(metadata_json.encode("utf-8"))
1062
+
1063
+ # Verify file was written correctly
1064
+ if os.path.getsize(metadata_path) == 0:
1065
+ raise ValueError("Failed to write metadata file (file size is 0)")
1033
1066
 
1034
1067
  if verbose:
1035
1068
  print("Uploading metadata file...")
1069
+ print(f"Metadata file size: {os.path.getsize(metadata_path)} bytes")
1036
1070
 
1037
1071
  # Upload the metadata file to IPFS
1038
1072
  metadata_cid_result = await self.upload_file(
@@ -1105,9 +1139,12 @@ class IPFSClient:
1105
1139
  if verbose:
1106
1140
  metadata_download_time = time.time() - start_time
1107
1141
  print(f"Metadata downloaded in {metadata_download_time:.2f} seconds")
1142
+ print(f"Metadata file size: {os.path.getsize(metadata_path)} bytes")
1108
1143
 
1109
- with open(metadata_path, "r") as f:
1110
- metadata = json.load(f)
1144
+ # Read using binary mode to avoid any encoding issues
1145
+ with open(metadata_path, "rb") as f:
1146
+ metadata_content = f.read().decode("utf-8")
1147
+ metadata = json.loads(metadata_content)
1111
1148
 
1112
1149
  # Step 2: Extract key information
1113
1150
  original_file = metadata["original_file"]
@@ -1386,6 +1423,7 @@ class IPFSClient:
1386
1423
  max_retries: int = 3,
1387
1424
  verbose: bool = True,
1388
1425
  progress_callback: Optional[Callable[[str, int, int], None]] = None,
1426
+ publish: bool = True,
1389
1427
  ) -> Dict[str, Any]:
1390
1428
  """
1391
1429
  Erasure code a file, upload the chunks to IPFS, and store in the Hippius marketplace.
@@ -1404,15 +1442,21 @@ class IPFSClient:
1404
1442
  verbose: Whether to print progress information
1405
1443
  progress_callback: Optional callback function for progress updates
1406
1444
  Function receives (stage_name, current, total)
1445
+ publish: Whether to publish to the blockchain (True) or just perform local
1446
+ erasure coding without publishing (False). When False, no password
1447
+ is needed for seed phrase access.
1407
1448
 
1408
1449
  Returns:
1409
- dict: Result including metadata CID and transaction hash
1450
+ dict: Result including metadata CID and transaction hash (if published)
1410
1451
 
1411
1452
  Raises:
1412
1453
  ValueError: If parameters are invalid
1413
1454
  RuntimeError: If processing fails
1414
1455
  """
1415
- # Step 1: Erasure code the file and upload chunks
1456
+ # Step 1: Create substrate client if we need it and are publishing
1457
+ if substrate_client is None and publish:
1458
+ substrate_client = SubstrateClient()
1459
+ # Step 2: Erasure code the file and upload chunks
1416
1460
  metadata = await self.erasure_code_file(
1417
1461
  file_path=file_path,
1418
1462
  k=k,
@@ -1424,50 +1468,52 @@ class IPFSClient:
1424
1468
  progress_callback=progress_callback,
1425
1469
  )
1426
1470
 
1427
- # Step 2: Create substrate client if we need it
1428
- if substrate_client is None:
1429
- substrate_client = SubstrateClient()
1430
-
1431
1471
  original_file = metadata["original_file"]
1432
1472
  metadata_cid = metadata["metadata_cid"]
1433
1473
 
1434
- # Create a list to hold all the file inputs (metadata + all chunks)
1435
- all_file_inputs = []
1474
+ # Initialize transaction hash variable
1475
+ tx_hash = None
1436
1476
 
1437
- # Step 3: Prepare metadata file for storage
1438
- if verbose:
1439
- print(
1440
- f"Preparing to store metadata and {len(metadata['chunks'])} chunks in the Hippius marketplace..."
1441
- )
1477
+ # Only proceed with blockchain storage if publish is True
1478
+ if publish:
1479
+ # Create a list to hold all the file inputs (metadata + all chunks)
1480
+ all_file_inputs = []
1442
1481
 
1443
- # Create a file input for the metadata file
1444
- metadata_file_input = FileInput(
1445
- file_hash=metadata_cid, file_name=f"{original_file['name']}.ec_metadata"
1446
- )
1447
- all_file_inputs.append(metadata_file_input)
1482
+ # Step 3: Prepare metadata file for storage
1483
+ if verbose:
1484
+ print(
1485
+ f"Preparing to store metadata and {len(metadata['chunks'])} chunks in the Hippius marketplace..."
1486
+ )
1448
1487
 
1449
- # Step 4: Add all chunks to the storage request
1450
- if verbose:
1451
- print("Adding all chunks to storage request...")
1452
-
1453
- for i, chunk in enumerate(metadata["chunks"]):
1454
- # Extract the CID string from the chunk's cid dictionary
1455
- chunk_cid = (
1456
- chunk["cid"]["cid"]
1457
- if isinstance(chunk["cid"], dict) and "cid" in chunk["cid"]
1458
- else chunk["cid"]
1488
+ # Create a file input for the metadata file
1489
+ metadata_file_input = FileInput(
1490
+ file_hash=metadata_cid, file_name=f"{original_file['name']}.ec_metadata"
1459
1491
  )
1460
- chunk_file_input = FileInput(file_hash=chunk_cid, file_name=chunk["name"])
1461
- all_file_inputs.append(chunk_file_input)
1492
+ all_file_inputs.append(metadata_file_input)
1462
1493
 
1463
- # Print progress for large numbers of chunks
1464
- if verbose and (i + 1) % 50 == 0:
1465
- print(
1466
- f" Prepared {i + 1}/{len(metadata['chunks'])} chunks for storage"
1494
+ # Step 4: Add all chunks to the storage request
1495
+ if verbose:
1496
+ print("Adding all chunks to storage request...")
1497
+
1498
+ for i, chunk in enumerate(metadata["chunks"]):
1499
+ # Extract the CID string from the chunk's cid dictionary
1500
+ chunk_cid = (
1501
+ chunk["cid"]["cid"]
1502
+ if isinstance(chunk["cid"], dict) and "cid" in chunk["cid"]
1503
+ else chunk["cid"]
1504
+ )
1505
+ chunk_file_input = FileInput(
1506
+ file_hash=chunk_cid, file_name=chunk["name"]
1467
1507
  )
1508
+ all_file_inputs.append(chunk_file_input)
1468
1509
 
1469
- # Step 5: Submit the storage request for all files
1470
- try:
1510
+ # Print progress for large numbers of chunks
1511
+ if verbose and (i + 1) % 50 == 0:
1512
+ print(
1513
+ f" Prepared {i + 1}/{len(metadata['chunks'])} chunks for storage"
1514
+ )
1515
+
1516
+ # Step 5: Submit the storage request for all files
1471
1517
  if verbose:
1472
1518
  print(
1473
1519
  f"Submitting storage request for 1 metadata file and {len(metadata['chunks'])} chunks..."
@@ -1476,7 +1522,6 @@ class IPFSClient:
1476
1522
  tx_hash = await substrate_client.storage_request(
1477
1523
  files=all_file_inputs, miner_ids=miner_ids
1478
1524
  )
1479
-
1480
1525
  if verbose:
1481
1526
  print("Successfully stored all files in marketplace!")
1482
1527
  print(f"Transaction hash: {tx_hash}")
@@ -1485,26 +1530,37 @@ class IPFSClient:
1485
1530
  f"Total files stored: {len(all_file_inputs)} (1 metadata + {len(metadata['chunks'])} chunks)"
1486
1531
  )
1487
1532
 
1488
- return {
1533
+ result = {
1489
1534
  "metadata": metadata,
1490
1535
  "metadata_cid": metadata_cid,
1491
1536
  "transaction_hash": tx_hash,
1492
1537
  "total_files_stored": len(all_file_inputs),
1493
1538
  }
1539
+ else:
1540
+ # Not publishing to blockchain (--no-publish flag used)
1541
+ if verbose:
1542
+ print("Not publishing to blockchain (--no-publish flag used)")
1543
+ print(f"Metadata CID: {metadata_cid}")
1544
+ print(f"Total chunks: {len(metadata['chunks'])}")
1494
1545
 
1495
- except Exception as e:
1496
- print(f"Error storing files in marketplace: {str(e)}")
1497
- # Return the metadata even if storage fails
1498
- return {"metadata": metadata, "metadata_cid": metadata_cid, "error": str(e)}
1546
+ result = {
1547
+ "metadata": metadata,
1548
+ "metadata_cid": metadata_cid,
1549
+ "total_files_stored": len(metadata["chunks"])
1550
+ + 1, # +1 for metadata file
1551
+ }
1552
+
1553
+ return result
1499
1554
 
1500
1555
  async def delete_file(
1501
1556
  self, cid: str, cancel_from_blockchain: bool = True
1502
1557
  ) -> Dict[str, Any]:
1503
1558
  """
1504
- Delete a file from IPFS and optionally cancel its storage on the blockchain.
1559
+ Delete a file or directory from IPFS and optionally cancel its storage on the blockchain.
1560
+ If deleting a directory, all files within the directory will be unpinned recursively.
1505
1561
 
1506
1562
  Args:
1507
- cid: Content Identifier (CID) of the file to delete
1563
+ cid: Content Identifier (CID) of the file/directory to delete
1508
1564
  cancel_from_blockchain: Whether to also cancel the storage request from the blockchain
1509
1565
 
1510
1566
  Returns:
@@ -1519,77 +1575,117 @@ class IPFSClient:
1519
1575
  "end_time": None,
1520
1576
  "duration_seconds": None,
1521
1577
  },
1578
+ "is_directory": False,
1579
+ "child_files": [],
1522
1580
  }
1523
1581
 
1524
- # First, unpin from IPFS
1582
+ # First check if this is a directory
1525
1583
  try:
1526
- print(f"Unpinning file from IPFS: {cid}")
1527
- try:
1528
- # Try to check if file exists in IPFS before unpinning
1529
- await self.exists(cid)
1530
- except Exception as exists_e:
1531
- print(f"ERROR: Error checking file existence: {exists_e}")
1584
+ ls_result = await self.client.ls(cid)
1585
+ is_directory = ls_result.get("is_directory", False)
1586
+ result["is_directory"] = is_directory
1587
+
1588
+ # If it's a directory, recursively unpin all contained files first
1589
+ if is_directory:
1590
+ print(f"Detected directory: {cid}")
1591
+ links = []
1592
+
1593
+ # Extract all links from the directory listing
1594
+ if "Objects" in ls_result and len(ls_result["Objects"]) > 0:
1595
+ for obj in ls_result["Objects"]:
1596
+ if "Links" in obj:
1597
+ links.extend(obj["Links"])
1598
+
1599
+ child_files = []
1600
+ # Unpin each item in the directory
1601
+ for link in links:
1602
+ link_hash = link.get("Hash")
1603
+ link_name = link.get("Name", "unknown")
1604
+ if link_hash:
1605
+ child_files.append({"cid": link_hash, "name": link_name})
1606
+ try:
1607
+ # Recursively delete if it's a subdirectory
1608
+ link_type = link.get("Type")
1609
+ if (
1610
+ link_type == 1
1611
+ or str(link_type) == "1"
1612
+ or link_type == "dir"
1613
+ ):
1614
+ # Recursive delete, but don't cancel from blockchain (we'll do that for parent)
1615
+ await self.delete_file(
1616
+ link_hash, cancel_from_blockchain=False
1617
+ )
1618
+ else:
1619
+ # Regular file unpin
1620
+ try:
1621
+ await self.client.unpin(link_hash)
1622
+ print(
1623
+ f"Unpinned file: {link_name} (CID: {link_hash})"
1624
+ )
1625
+ except Exception as unpin_error:
1626
+ # Just note the error but don't let it stop the whole process
1627
+ # This is common with IPFS servers that may return 500 errors for
1628
+ # unpinning content that was never explicitly pinned
1629
+ print(
1630
+ f"Note: Could not unpin {link_name}: {str(unpin_error).split('For more information')[0]}"
1631
+ )
1632
+ except Exception as e:
1633
+ print(
1634
+ f"Warning: Problem processing child item {link_name}: {str(e).split('For more information')[0]}"
1635
+ )
1532
1636
 
1637
+ # Record the child files that were processed
1638
+ result["child_files"] = child_files
1639
+ except Exception as e:
1640
+ print(f"Warning: Failed to check if CID is a directory: {e}")
1641
+ # Continue with regular file unpin
1642
+
1643
+ # Now unpin the main file/directory
1644
+ try:
1645
+ print(f"Unpinning from IPFS: {cid}")
1533
1646
  unpin_result = await self.client.unpin(cid)
1534
1647
  result["unpin_result"] = unpin_result
1648
+ result["success"] = True
1535
1649
  print("Successfully unpinned from IPFS")
1536
1650
  except Exception as e:
1537
- print(f"Warning: Failed to unpin file from IPFS: {e}")
1538
- raise
1651
+ # Handle 500 errors from IPFS server gracefully - they often occur
1652
+ # when the content wasn't explicitly pinned or was already unpinned
1653
+ error_str = str(e)
1654
+ if "500 Internal Server Error" in error_str:
1655
+ print(
1656
+ f"Note: IPFS server reported content may already be unpinned: {cid}"
1657
+ )
1658
+ result["unpin_result"] = {"Pins": [cid]} # Simulate successful unpin
1659
+ result["success"] = True
1660
+ else:
1661
+ print(
1662
+ f"Warning: Failed to unpin from IPFS: {error_str.split('For more information')[0]}"
1663
+ )
1664
+ result["success"] = False
1539
1665
 
1540
1666
  # Then, if requested, cancel from blockchain
1541
1667
  if cancel_from_blockchain:
1542
1668
  try:
1543
- # Create a substrate client
1544
- print(f"DEBUG: Creating SubstrateClient for blockchain cancellation...")
1545
1669
  substrate_client = SubstrateClient()
1546
- print(
1547
- f"DEBUG: Substrate client created with URL: {substrate_client.url}"
1548
- )
1549
- print(f"DEBUG: Calling cancel_storage_request with CID: {cid}")
1550
-
1551
- tx_hash = await substrate_client.cancel_storage_request(cid)
1552
- print(f"DEBUG: Received transaction hash: {tx_hash}")
1553
-
1554
- # Check the return value - special cases for when blockchain cancellation isn't available
1555
- if tx_hash == "no-blockchain-cancellation-available":
1556
- print(
1557
- "Blockchain cancellation not available, but IPFS unpinning was successful"
1558
- )
1559
- result["blockchain_result"] = {
1560
- "status": "not_available",
1561
- "message": "Blockchain cancellation not available, but IPFS unpinning was successful",
1562
- }
1563
- elif tx_hash.startswith("ipfs-unpinned-only"):
1564
- error_msg = tx_hash.replace("ipfs-unpinned-only-", "")
1670
+ await substrate_client.cancel_storage_request(cid)
1671
+ print("Successfully cancelled storage from blockchain")
1672
+ result["blockchain_result"] = {"success": True}
1673
+ except Exception as e:
1674
+ # Handle the case where the CID is not in storage requests
1675
+ error_str = str(e)
1676
+ if "not found in storage requests" in error_str:
1565
1677
  print(
1566
- f"IPFS unpinning successful, but blockchain cancellation failed: {error_msg}"
1678
+ "Note: Content was not found in blockchain storage requests (may already be deleted)"
1567
1679
  )
1568
1680
  result["blockchain_result"] = {
1569
- "status": "failed",
1570
- "error": error_msg,
1571
- "message": "IPFS unpinning successful, but blockchain cancellation failed",
1681
+ "success": True,
1682
+ "already_deleted": True,
1572
1683
  }
1573
1684
  else:
1574
- # Standard successful transaction
1575
- result["blockchain_result"] = {
1576
- "transaction_hash": tx_hash,
1577
- "status": "success",
1578
- }
1579
- print(f"Successfully canceled storage request from blockchain")
1580
- print(
1581
- f"DEBUG: Blockchain cancellation succeeded with transaction hash: {tx_hash}"
1582
- )
1583
- except Exception as e:
1584
- print(f"Warning: Failed to cancel storage from blockchain: {e}")
1585
- print(
1586
- f"DEBUG: Blockchain cancellation exception: {type(e).__name__}: {str(e)}"
1587
- )
1588
- if hasattr(e, "__dict__"):
1589
- print(f"DEBUG: Exception attributes: {e.__dict__}")
1590
- result["blockchain_error"] = str(e)
1685
+ print(f"Warning: Error cancelling from blockchain: {error_str}")
1686
+ result["blockchain_result"] = {"success": False, "error": error_str}
1591
1687
 
1592
- # Calculate timing
1688
+ # Update timing information
1593
1689
  result["timing"]["end_time"] = time.time()
1594
1690
  result["timing"]["duration_seconds"] = (
1595
1691
  result["timing"]["end_time"] - result["timing"]["start_time"]
@@ -1602,7 +1698,7 @@ class IPFSClient:
1602
1698
  metadata_cid: str,
1603
1699
  cancel_from_blockchain: bool = True,
1604
1700
  parallel_limit: int = 20,
1605
- ) -> Dict[str, Any]:
1701
+ ) -> bool:
1606
1702
  """
1607
1703
  Delete an erasure-coded file, including all its chunks in parallel.
1608
1704
 
@@ -1612,219 +1708,84 @@ class IPFSClient:
1612
1708
  parallel_limit: Maximum number of concurrent deletion operations
1613
1709
 
1614
1710
  Returns:
1615
- Dict containing the result of the operation
1711
+ bool: True if the deletion was successful, False otherwise
1616
1712
  """
1617
- result = {
1618
- "metadata_cid": metadata_cid,
1619
- "deleted_chunks": [],
1620
- "failed_chunks": [],
1621
- "blockchain_result": None,
1622
- "timing": {
1623
- "start_time": time.time(),
1624
- "end_time": None,
1625
- "duration_seconds": None,
1626
- },
1627
- }
1628
1713
 
1629
- # Track deletions for reporting
1630
- deleted_chunks_lock = asyncio.Lock()
1631
- failed_chunks_lock = asyncio.Lock()
1714
+ # Try to download and process metadata file and chunks
1715
+ ipfs_failure = False
1716
+ metadata_error = False
1632
1717
 
1633
- # First, get the metadata to find all chunks
1634
1718
  try:
1635
- print(f"Downloading metadata file (CID: {metadata_cid})...")
1636
- start_time = time.time()
1637
- metadata_content = await self.client.cat(metadata_cid)
1638
- metadata = json.loads(metadata_content.decode("utf-8"))
1639
- metadata_download_time = time.time() - start_time
1640
-
1641
- print(f"Metadata downloaded in {metadata_download_time:.2f} seconds")
1642
-
1643
- # Extract chunk CIDs
1644
- chunks = []
1645
- total_chunks = 0
1646
-
1647
- for chunk_data in metadata.get("chunks", []):
1648
- for ec_chunk in chunk_data.get("ec_chunks", []):
1649
- chunk_cid = ec_chunk.get("cid")
1650
- if chunk_cid:
1651
- chunks.append(chunk_cid)
1652
- total_chunks += 1
1653
-
1654
- print(f"Found {total_chunks} chunks to delete")
1719
+ # First download the metadata to get chunk CIDs
1720
+ try:
1721
+ metadata_result = await self.cat(metadata_cid)
1722
+ metadata_json = json.loads(metadata_result["content"].decode("utf-8"))
1723
+ chunks = metadata_json.get("chunks", [])
1724
+ except json.JSONDecodeError:
1725
+ # If we can't parse the metadata JSON, record the error but continue
1726
+ metadata_error = True
1727
+ # Continue with empty chunks so we can at least try to unpin the metadata file
1728
+ chunks = []
1729
+ except Exception:
1730
+ # Any other metadata error
1731
+ metadata_error = True
1732
+ chunks = []
1733
+
1734
+ # Extract all chunk CIDs
1735
+ chunk_cids = []
1736
+ for chunk in chunks:
1737
+ chunk_cid = chunk.get("cid", {})
1738
+ if isinstance(chunk_cid, dict) and "cid" in chunk_cid:
1739
+ chunk_cids.append(chunk_cid["cid"])
1740
+ elif isinstance(chunk_cid, str):
1741
+ chunk_cids.append(chunk_cid)
1655
1742
 
1656
1743
  # Create a semaphore to limit concurrent operations
1657
- sem = asyncio.Semaphore(parallel_limit)
1744
+ semaphore = asyncio.Semaphore(parallel_limit)
1658
1745
 
1659
- # Define the chunk deletion function
1660
- async def delete_chunk(chunk_cid):
1661
- async with sem:
1746
+ # Define the unpin task for each chunk with error handling
1747
+ async def unpin_chunk(cid):
1748
+ async with semaphore:
1662
1749
  try:
1663
- print(f"Unpinning chunk: {chunk_cid}")
1664
- await self.client.unpin(chunk_cid)
1665
-
1666
- # Record success
1667
- async with deleted_chunks_lock:
1668
- result["deleted_chunks"].append(chunk_cid)
1669
-
1670
- # Cancel from blockchain if requested
1671
- if cancel_from_blockchain:
1672
- try:
1673
- substrate_client = SubstrateClient()
1674
- tx_hash = await substrate_client.cancel_storage_request(
1675
- chunk_cid
1676
- )
1677
-
1678
- # Add blockchain result
1679
- if "chunk_results" not in result["blockchain_result"]:
1680
- result["blockchain_result"] = {}
1681
- result["blockchain_result"]["chunk_results"] = []
1682
-
1683
- # Handle special return values from cancel_storage_request
1684
- if tx_hash == "no-blockchain-cancellation-available":
1685
- result["blockchain_result"]["chunk_results"].append(
1686
- {
1687
- "cid": chunk_cid,
1688
- "status": "not_available",
1689
- "message": "Blockchain cancellation not available",
1690
- }
1691
- )
1692
- elif tx_hash.startswith("ipfs-unpinned-only"):
1693
- error_msg = tx_hash.replace(
1694
- "ipfs-unpinned-only-", ""
1695
- )
1696
- result["blockchain_result"]["chunk_results"].append(
1697
- {
1698
- "cid": chunk_cid,
1699
- "status": "failed",
1700
- "error": error_msg,
1701
- }
1702
- )
1703
- else:
1704
- # Standard successful transaction
1705
- result["blockchain_result"]["chunk_results"].append(
1706
- {
1707
- "cid": chunk_cid,
1708
- "transaction_hash": tx_hash,
1709
- "status": "success",
1710
- }
1711
- )
1712
- except Exception as e:
1713
- print(
1714
- f"Warning: Failed to cancel blockchain storage for chunk {chunk_cid}: {e}"
1715
- )
1716
-
1717
- if "chunk_results" not in result["blockchain_result"]:
1718
- result["blockchain_result"] = {}
1719
- result["blockchain_result"]["chunk_results"] = []
1720
-
1721
- result["blockchain_result"]["chunk_results"].append(
1722
- {
1723
- "cid": chunk_cid,
1724
- "error": str(e),
1725
- "status": "failed",
1726
- }
1727
- )
1728
-
1729
- return True
1730
- except Exception as e:
1731
- error_msg = f"Failed to delete chunk {chunk_cid}: {e}"
1732
- print(f"Warning: {error_msg}")
1733
-
1734
- # Record failure
1735
- async with failed_chunks_lock:
1736
- result["failed_chunks"].append(
1737
- {"cid": chunk_cid, "error": str(e)}
1738
- )
1739
-
1740
- return False
1741
-
1742
- # Start deleting chunks in parallel
1743
- print(
1744
- f"Starting parallel deletion of {total_chunks} chunks with max {parallel_limit} concurrent operations"
1745
- )
1746
- delete_tasks = [delete_chunk(cid) for cid in chunks]
1747
- await asyncio.gather(*delete_tasks)
1748
-
1749
- # Delete the metadata file itself
1750
- print(f"Unpinning metadata file: {metadata_cid}")
1751
- response = await self.client.unpin(metadata_cid)
1752
-
1753
- print(">>>", response)
1754
- raise SystemExit
1755
-
1756
- # Cancel metadata from blockchain if requested
1757
- if cancel_from_blockchain:
1758
- try:
1759
- print(f"Canceling blockchain storage request for metadata file...")
1760
- substrate_client = SubstrateClient()
1761
- tx_hash = await substrate_client.cancel_storage_request(
1762
- metadata_cid
1763
- )
1764
-
1765
- # Handle special return values from cancel_storage_request
1766
- if tx_hash == "no-blockchain-cancellation-available":
1767
- print(
1768
- "Blockchain cancellation not available for metadata, but IPFS unpinning was successful"
1769
- )
1770
- result["blockchain_result"] = {
1771
- "status": "not_available",
1772
- "message": "Blockchain cancellation not available, but IPFS unpinning was successful",
1773
- }
1774
- elif tx_hash.startswith("ipfs-unpinned-only"):
1775
- error_msg = tx_hash.replace("ipfs-unpinned-only-", "")
1776
- print(
1777
- f"IPFS unpinning successful, but blockchain cancellation failed for metadata: {error_msg}"
1778
- )
1779
- result["blockchain_result"] = {
1780
- "status": "failed",
1781
- "error": error_msg,
1782
- "message": "IPFS unpinning successful, but blockchain cancellation failed",
1783
- }
1784
- else:
1785
- # Standard successful transaction
1786
- result["blockchain_result"] = {
1787
- "metadata_transaction_hash": tx_hash,
1788
- "status": "success",
1789
- }
1790
- print(
1791
- f"Successfully canceled blockchain storage for metadata file"
1792
- )
1793
- except Exception as e:
1794
- print(
1795
- f"Warning: Failed to cancel blockchain storage for metadata file: {e}"
1796
- )
1797
-
1798
- if not result["blockchain_result"]:
1799
- result["blockchain_result"] = {}
1800
-
1801
- result["blockchain_result"]["metadata_error"] = str(e)
1802
- result["blockchain_result"]["status"] = "failed"
1803
-
1804
- # Calculate and record timing information
1805
- end_time = time.time()
1806
- duration = end_time - result["timing"]["start_time"]
1807
-
1808
- result["timing"]["end_time"] = end_time
1809
- result["timing"]["duration_seconds"] = duration
1810
-
1811
- deleted_count = len(result["deleted_chunks"])
1812
- failed_count = len(result["failed_chunks"])
1813
-
1814
- print(f"Deletion complete in {duration:.2f} seconds!")
1815
- print(f"Successfully deleted: {deleted_count}/{total_chunks} chunks")
1816
-
1817
- if failed_count > 0:
1818
- print(f"Failed to delete: {failed_count}/{total_chunks} chunks")
1750
+ await self.client.unpin(cid)
1751
+ return {"success": True, "cid": cid}
1752
+ except Exception:
1753
+ # Record failure but continue with other chunks
1754
+ return {"success": False, "cid": cid}
1755
+
1756
+ # Unpin all chunks in parallel
1757
+ if chunk_cids:
1758
+ unpin_tasks = [unpin_chunk(cid) for cid in chunk_cids]
1759
+ results = await asyncio.gather(*unpin_tasks)
1760
+
1761
+ # Count failures
1762
+ failures = [r for r in results if not r["success"]]
1763
+ if failures:
1764
+ ipfs_failure = True
1765
+ except Exception:
1766
+ # If we can't process chunks at all, record the failure
1767
+ ipfs_failure = True
1768
+
1769
+ # Unpin the metadata file itself, regardless of whether we could process chunks
1770
+ try:
1771
+ await self.client.unpin(metadata_cid)
1772
+ except Exception:
1773
+ # Record the failure but continue with blockchain cancellation
1774
+ ipfs_failure = True
1819
1775
 
1820
- return result
1821
- except Exception as e:
1822
- # Record end time even if there was an error
1823
- result["timing"]["end_time"] = time.time()
1824
- result["timing"]["duration_seconds"] = (
1825
- result["timing"]["end_time"] - result["timing"]["start_time"]
1826
- )
1776
+ # Handle blockchain cancellation if requested
1777
+ if cancel_from_blockchain:
1778
+ # Create a substrate client
1779
+ substrate_client = SubstrateClient()
1827
1780
 
1828
- error_msg = f"Error deleting erasure-coded file: {e}"
1829
- print(f"Error: {error_msg}")
1830
- raise RuntimeError(error_msg)
1781
+ # This will raise appropriate exceptions if it fails:
1782
+ # - HippiusAlreadyDeletedError if already deleted
1783
+ # - HippiusFailedSubstrateDelete if transaction fails
1784
+ # - Other exceptions for other failures
1785
+ await substrate_client.cancel_storage_request(metadata_cid)
1786
+
1787
+ # If we get here, either:
1788
+ # 1. Blockchain cancellation succeeded (if requested)
1789
+ # 2. We weren't doing blockchain cancellation
1790
+ # In either case, we report success
1791
+ return True