hippius 0.2.3__py3-none-any.whl → 0.2.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hippius-0.2.3.dist-info → hippius-0.2.5.dist-info}/METADATA +207 -151
- hippius-0.2.5.dist-info/RECORD +17 -0
- hippius_sdk/__init__.py +1 -1
- hippius_sdk/cli.py +18 -1
- hippius_sdk/cli_assets.py +8 -6
- hippius_sdk/cli_handlers.py +796 -185
- hippius_sdk/cli_parser.py +25 -0
- hippius_sdk/client.py +19 -17
- hippius_sdk/config.py +108 -141
- hippius_sdk/errors.py +77 -0
- hippius_sdk/ipfs.py +301 -340
- hippius_sdk/ipfs_core.py +209 -9
- hippius_sdk/substrate.py +105 -27
- hippius-0.2.3.dist-info/RECORD +0 -16
- {hippius-0.2.3.dist-info → hippius-0.2.5.dist-info}/WHEEL +0 -0
- {hippius-0.2.3.dist-info → hippius-0.2.5.dist-info}/entry_points.txt +0 -0
hippius_sdk/ipfs.py
CHANGED
@@ -16,6 +16,13 @@ import httpx
|
|
16
16
|
import requests
|
17
17
|
|
18
18
|
from hippius_sdk.config import get_config_value, get_encryption_key
|
19
|
+
from hippius_sdk.errors import (
|
20
|
+
HippiusAlreadyDeletedError,
|
21
|
+
HippiusFailedIPFSUnpin,
|
22
|
+
HippiusFailedSubstrateDelete,
|
23
|
+
HippiusIPFSConnectionError,
|
24
|
+
HippiusMetadataError,
|
25
|
+
)
|
19
26
|
from hippius_sdk.ipfs_core import AsyncIPFSClient
|
20
27
|
from hippius_sdk.substrate import FileInput, SubstrateClient
|
21
28
|
from hippius_sdk.utils import format_cid, format_size
|
@@ -439,10 +446,11 @@ class IPFSClient:
|
|
439
446
|
) -> Dict[str, Any]:
|
440
447
|
"""
|
441
448
|
Download a file from IPFS with optional decryption.
|
449
|
+
Supports downloading directories - in that case, a directory structure will be created.
|
442
450
|
|
443
451
|
Args:
|
444
452
|
cid: Content Identifier (CID) of the file to download
|
445
|
-
output_path: Path where the downloaded file will be saved
|
453
|
+
output_path: Path where the downloaded file/directory will be saved
|
446
454
|
decrypt: Whether to decrypt the file (overrides default)
|
447
455
|
max_retries: Maximum number of retry attempts (default: 3)
|
448
456
|
|
@@ -454,6 +462,7 @@ class IPFSClient:
|
|
454
462
|
- size_formatted: Human-readable file size
|
455
463
|
- elapsed_seconds: Time taken for the download in seconds
|
456
464
|
- decrypted: Whether the file was decrypted
|
465
|
+
- is_directory: Whether the download was a directory
|
457
466
|
|
458
467
|
Raises:
|
459
468
|
requests.RequestException: If the download fails
|
@@ -461,6 +470,45 @@ class IPFSClient:
|
|
461
470
|
"""
|
462
471
|
start_time = time.time()
|
463
472
|
|
473
|
+
# Use the improved ls function to properly detect directories
|
474
|
+
is_directory = False
|
475
|
+
try:
|
476
|
+
# The ls function now properly detects directories
|
477
|
+
ls_result = await self.client.ls(cid)
|
478
|
+
is_directory = ls_result.get("is_directory", False)
|
479
|
+
except Exception:
|
480
|
+
# If ls fails, we'll proceed as if it's a file
|
481
|
+
pass
|
482
|
+
|
483
|
+
# If it's a directory, handle it differently
|
484
|
+
if is_directory:
|
485
|
+
# For directories, we don't need to decrypt each file during the initial download
|
486
|
+
# We'll use the AsyncIPFSClient's download_directory method directly
|
487
|
+
try:
|
488
|
+
await self.client.download_directory(cid, output_path)
|
489
|
+
|
490
|
+
# Calculate the total size of the directory
|
491
|
+
total_size = 0
|
492
|
+
for root, _, files in os.walk(output_path):
|
493
|
+
for file in files:
|
494
|
+
file_path = os.path.join(root, file)
|
495
|
+
total_size += os.path.getsize(file_path)
|
496
|
+
|
497
|
+
elapsed_time = time.time() - start_time
|
498
|
+
|
499
|
+
return {
|
500
|
+
"success": True,
|
501
|
+
"output_path": output_path,
|
502
|
+
"size_bytes": total_size,
|
503
|
+
"size_formatted": self.format_size(total_size),
|
504
|
+
"elapsed_seconds": round(elapsed_time, 2),
|
505
|
+
"decrypted": False, # Directories aren't decrypted as a whole
|
506
|
+
"is_directory": True,
|
507
|
+
}
|
508
|
+
except Exception as e:
|
509
|
+
raise RuntimeError(f"Failed to download directory: {str(e)}")
|
510
|
+
|
511
|
+
# For regular files, use the existing logic
|
464
512
|
# Determine if we should decrypt
|
465
513
|
should_decrypt = self.encrypt_by_default if decrypt is None else decrypt
|
466
514
|
|
@@ -482,38 +530,11 @@ class IPFSClient:
|
|
482
530
|
else:
|
483
531
|
download_path = output_path
|
484
532
|
|
485
|
-
|
486
|
-
|
487
|
-
while retries < max_retries:
|
488
|
-
try:
|
489
|
-
# Download the file
|
490
|
-
url = f"{self.gateway}/ipfs/{cid}"
|
491
|
-
response = requests.get(url, stream=True)
|
492
|
-
response.raise_for_status()
|
493
|
-
|
494
|
-
os.makedirs(
|
495
|
-
os.path.dirname(os.path.abspath(download_path)), exist_ok=True
|
496
|
-
)
|
497
|
-
|
498
|
-
with open(download_path, "wb") as f:
|
499
|
-
for chunk in response.iter_content(chunk_size=8192):
|
500
|
-
f.write(chunk)
|
501
|
-
|
502
|
-
# If we reach here, download was successful
|
503
|
-
break
|
533
|
+
await self.client.download_file(cid, download_path)
|
534
|
+
download_success = True
|
504
535
|
|
505
|
-
|
506
|
-
|
507
|
-
retries += 1
|
508
|
-
|
509
|
-
if retries < max_retries:
|
510
|
-
wait_time = 2**retries # Exponential backoff: 2, 4, 8 seconds
|
511
|
-
print(f"Download attempt {retries} failed: {str(e)}")
|
512
|
-
print(f"Retrying in {wait_time} seconds...")
|
513
|
-
time.sleep(wait_time)
|
514
|
-
else:
|
515
|
-
# Raise the last error if we've exhausted all retries
|
516
|
-
raise
|
536
|
+
if not download_success:
|
537
|
+
raise RuntimeError("Failed to download file after multiple attempts")
|
517
538
|
|
518
539
|
# Decrypt if needed
|
519
540
|
if should_decrypt:
|
@@ -548,6 +569,7 @@ class IPFSClient:
|
|
548
569
|
"size_formatted": self.format_size(file_size_bytes),
|
549
570
|
"elapsed_seconds": round(elapsed_time, 2),
|
550
571
|
"decrypted": should_decrypt,
|
572
|
+
"is_directory": False,
|
551
573
|
}
|
552
574
|
|
553
575
|
finally:
|
@@ -816,6 +838,9 @@ class IPFSClient:
|
|
816
838
|
file_data = self.encrypt_data(file_data)
|
817
839
|
|
818
840
|
# Step 2: Split the file into chunks for erasure coding
|
841
|
+
chunk_size = int(chunk_size)
|
842
|
+
chunk_size = max(1, chunk_size) # Ensure it's at least 1 byte
|
843
|
+
|
819
844
|
chunks = []
|
820
845
|
chunk_positions = []
|
821
846
|
for i in range(0, len(file_data), chunk_size):
|
@@ -825,7 +850,7 @@ class IPFSClient:
|
|
825
850
|
|
826
851
|
# Pad the last chunk if necessary
|
827
852
|
if chunks and len(chunks[-1]) < chunk_size:
|
828
|
-
pad_size = chunk_size - len(chunks[-1])
|
853
|
+
pad_size = int(chunk_size - len(chunks[-1]))
|
829
854
|
chunks[-1] = chunks[-1] + b"\0" * pad_size
|
830
855
|
|
831
856
|
# If we don't have enough chunks for the requested parameters, adjust
|
@@ -1028,11 +1053,20 @@ class IPFSClient:
|
|
1028
1053
|
|
1029
1054
|
# Step 5: Create and upload the metadata file
|
1030
1055
|
metadata_path = os.path.join(temp_dir, f"{file_id}_metadata.json")
|
1031
|
-
|
1032
|
-
|
1056
|
+
|
1057
|
+
# Use binary mode to avoid any platform-specific text encoding issues
|
1058
|
+
with open(metadata_path, "wb") as f:
|
1059
|
+
# Encode the JSON with UTF-8 encoding explicitly
|
1060
|
+
metadata_json = json.dumps(metadata, indent=2, ensure_ascii=False)
|
1061
|
+
f.write(metadata_json.encode("utf-8"))
|
1062
|
+
|
1063
|
+
# Verify file was written correctly
|
1064
|
+
if os.path.getsize(metadata_path) == 0:
|
1065
|
+
raise ValueError("Failed to write metadata file (file size is 0)")
|
1033
1066
|
|
1034
1067
|
if verbose:
|
1035
1068
|
print("Uploading metadata file...")
|
1069
|
+
print(f"Metadata file size: {os.path.getsize(metadata_path)} bytes")
|
1036
1070
|
|
1037
1071
|
# Upload the metadata file to IPFS
|
1038
1072
|
metadata_cid_result = await self.upload_file(
|
@@ -1105,9 +1139,12 @@ class IPFSClient:
|
|
1105
1139
|
if verbose:
|
1106
1140
|
metadata_download_time = time.time() - start_time
|
1107
1141
|
print(f"Metadata downloaded in {metadata_download_time:.2f} seconds")
|
1142
|
+
print(f"Metadata file size: {os.path.getsize(metadata_path)} bytes")
|
1108
1143
|
|
1109
|
-
|
1110
|
-
|
1144
|
+
# Read using binary mode to avoid any encoding issues
|
1145
|
+
with open(metadata_path, "rb") as f:
|
1146
|
+
metadata_content = f.read().decode("utf-8")
|
1147
|
+
metadata = json.loads(metadata_content)
|
1111
1148
|
|
1112
1149
|
# Step 2: Extract key information
|
1113
1150
|
original_file = metadata["original_file"]
|
@@ -1386,6 +1423,7 @@ class IPFSClient:
|
|
1386
1423
|
max_retries: int = 3,
|
1387
1424
|
verbose: bool = True,
|
1388
1425
|
progress_callback: Optional[Callable[[str, int, int], None]] = None,
|
1426
|
+
publish: bool = True,
|
1389
1427
|
) -> Dict[str, Any]:
|
1390
1428
|
"""
|
1391
1429
|
Erasure code a file, upload the chunks to IPFS, and store in the Hippius marketplace.
|
@@ -1404,15 +1442,21 @@ class IPFSClient:
|
|
1404
1442
|
verbose: Whether to print progress information
|
1405
1443
|
progress_callback: Optional callback function for progress updates
|
1406
1444
|
Function receives (stage_name, current, total)
|
1445
|
+
publish: Whether to publish to the blockchain (True) or just perform local
|
1446
|
+
erasure coding without publishing (False). When False, no password
|
1447
|
+
is needed for seed phrase access.
|
1407
1448
|
|
1408
1449
|
Returns:
|
1409
|
-
dict: Result including metadata CID and transaction hash
|
1450
|
+
dict: Result including metadata CID and transaction hash (if published)
|
1410
1451
|
|
1411
1452
|
Raises:
|
1412
1453
|
ValueError: If parameters are invalid
|
1413
1454
|
RuntimeError: If processing fails
|
1414
1455
|
"""
|
1415
|
-
# Step 1:
|
1456
|
+
# Step 1: Create substrate client if we need it and are publishing
|
1457
|
+
if substrate_client is None and publish:
|
1458
|
+
substrate_client = SubstrateClient()
|
1459
|
+
# Step 2: Erasure code the file and upload chunks
|
1416
1460
|
metadata = await self.erasure_code_file(
|
1417
1461
|
file_path=file_path,
|
1418
1462
|
k=k,
|
@@ -1424,50 +1468,52 @@ class IPFSClient:
|
|
1424
1468
|
progress_callback=progress_callback,
|
1425
1469
|
)
|
1426
1470
|
|
1427
|
-
# Step 2: Create substrate client if we need it
|
1428
|
-
if substrate_client is None:
|
1429
|
-
substrate_client = SubstrateClient()
|
1430
|
-
|
1431
1471
|
original_file = metadata["original_file"]
|
1432
1472
|
metadata_cid = metadata["metadata_cid"]
|
1433
1473
|
|
1434
|
-
#
|
1435
|
-
|
1474
|
+
# Initialize transaction hash variable
|
1475
|
+
tx_hash = None
|
1436
1476
|
|
1437
|
-
#
|
1438
|
-
if
|
1439
|
-
|
1440
|
-
|
1441
|
-
)
|
1477
|
+
# Only proceed with blockchain storage if publish is True
|
1478
|
+
if publish:
|
1479
|
+
# Create a list to hold all the file inputs (metadata + all chunks)
|
1480
|
+
all_file_inputs = []
|
1442
1481
|
|
1443
|
-
|
1444
|
-
|
1445
|
-
|
1446
|
-
|
1447
|
-
|
1482
|
+
# Step 3: Prepare metadata file for storage
|
1483
|
+
if verbose:
|
1484
|
+
print(
|
1485
|
+
f"Preparing to store metadata and {len(metadata['chunks'])} chunks in the Hippius marketplace..."
|
1486
|
+
)
|
1448
1487
|
|
1449
|
-
|
1450
|
-
|
1451
|
-
|
1452
|
-
|
1453
|
-
for i, chunk in enumerate(metadata["chunks"]):
|
1454
|
-
# Extract the CID string from the chunk's cid dictionary
|
1455
|
-
chunk_cid = (
|
1456
|
-
chunk["cid"]["cid"]
|
1457
|
-
if isinstance(chunk["cid"], dict) and "cid" in chunk["cid"]
|
1458
|
-
else chunk["cid"]
|
1488
|
+
# Create a file input for the metadata file
|
1489
|
+
metadata_file_input = FileInput(
|
1490
|
+
file_hash=metadata_cid, file_name=f"{original_file['name']}.ec_metadata"
|
1459
1491
|
)
|
1460
|
-
|
1461
|
-
all_file_inputs.append(chunk_file_input)
|
1492
|
+
all_file_inputs.append(metadata_file_input)
|
1462
1493
|
|
1463
|
-
#
|
1464
|
-
if verbose
|
1465
|
-
print(
|
1466
|
-
|
1494
|
+
# Step 4: Add all chunks to the storage request
|
1495
|
+
if verbose:
|
1496
|
+
print("Adding all chunks to storage request...")
|
1497
|
+
|
1498
|
+
for i, chunk in enumerate(metadata["chunks"]):
|
1499
|
+
# Extract the CID string from the chunk's cid dictionary
|
1500
|
+
chunk_cid = (
|
1501
|
+
chunk["cid"]["cid"]
|
1502
|
+
if isinstance(chunk["cid"], dict) and "cid" in chunk["cid"]
|
1503
|
+
else chunk["cid"]
|
1504
|
+
)
|
1505
|
+
chunk_file_input = FileInput(
|
1506
|
+
file_hash=chunk_cid, file_name=chunk["name"]
|
1467
1507
|
)
|
1508
|
+
all_file_inputs.append(chunk_file_input)
|
1468
1509
|
|
1469
|
-
|
1470
|
-
|
1510
|
+
# Print progress for large numbers of chunks
|
1511
|
+
if verbose and (i + 1) % 50 == 0:
|
1512
|
+
print(
|
1513
|
+
f" Prepared {i + 1}/{len(metadata['chunks'])} chunks for storage"
|
1514
|
+
)
|
1515
|
+
|
1516
|
+
# Step 5: Submit the storage request for all files
|
1471
1517
|
if verbose:
|
1472
1518
|
print(
|
1473
1519
|
f"Submitting storage request for 1 metadata file and {len(metadata['chunks'])} chunks..."
|
@@ -1476,7 +1522,6 @@ class IPFSClient:
|
|
1476
1522
|
tx_hash = await substrate_client.storage_request(
|
1477
1523
|
files=all_file_inputs, miner_ids=miner_ids
|
1478
1524
|
)
|
1479
|
-
|
1480
1525
|
if verbose:
|
1481
1526
|
print("Successfully stored all files in marketplace!")
|
1482
1527
|
print(f"Transaction hash: {tx_hash}")
|
@@ -1485,26 +1530,37 @@ class IPFSClient:
|
|
1485
1530
|
f"Total files stored: {len(all_file_inputs)} (1 metadata + {len(metadata['chunks'])} chunks)"
|
1486
1531
|
)
|
1487
1532
|
|
1488
|
-
|
1533
|
+
result = {
|
1489
1534
|
"metadata": metadata,
|
1490
1535
|
"metadata_cid": metadata_cid,
|
1491
1536
|
"transaction_hash": tx_hash,
|
1492
1537
|
"total_files_stored": len(all_file_inputs),
|
1493
1538
|
}
|
1539
|
+
else:
|
1540
|
+
# Not publishing to blockchain (--no-publish flag used)
|
1541
|
+
if verbose:
|
1542
|
+
print("Not publishing to blockchain (--no-publish flag used)")
|
1543
|
+
print(f"Metadata CID: {metadata_cid}")
|
1544
|
+
print(f"Total chunks: {len(metadata['chunks'])}")
|
1494
1545
|
|
1495
|
-
|
1496
|
-
|
1497
|
-
|
1498
|
-
|
1546
|
+
result = {
|
1547
|
+
"metadata": metadata,
|
1548
|
+
"metadata_cid": metadata_cid,
|
1549
|
+
"total_files_stored": len(metadata["chunks"])
|
1550
|
+
+ 1, # +1 for metadata file
|
1551
|
+
}
|
1552
|
+
|
1553
|
+
return result
|
1499
1554
|
|
1500
1555
|
async def delete_file(
|
1501
1556
|
self, cid: str, cancel_from_blockchain: bool = True
|
1502
1557
|
) -> Dict[str, Any]:
|
1503
1558
|
"""
|
1504
|
-
Delete a file from IPFS and optionally cancel its storage on the blockchain.
|
1559
|
+
Delete a file or directory from IPFS and optionally cancel its storage on the blockchain.
|
1560
|
+
If deleting a directory, all files within the directory will be unpinned recursively.
|
1505
1561
|
|
1506
1562
|
Args:
|
1507
|
-
cid: Content Identifier (CID) of the file to delete
|
1563
|
+
cid: Content Identifier (CID) of the file/directory to delete
|
1508
1564
|
cancel_from_blockchain: Whether to also cancel the storage request from the blockchain
|
1509
1565
|
|
1510
1566
|
Returns:
|
@@ -1519,77 +1575,117 @@ class IPFSClient:
|
|
1519
1575
|
"end_time": None,
|
1520
1576
|
"duration_seconds": None,
|
1521
1577
|
},
|
1578
|
+
"is_directory": False,
|
1579
|
+
"child_files": [],
|
1522
1580
|
}
|
1523
1581
|
|
1524
|
-
# First
|
1582
|
+
# First check if this is a directory
|
1525
1583
|
try:
|
1526
|
-
|
1527
|
-
|
1528
|
-
|
1529
|
-
|
1530
|
-
|
1531
|
-
|
1584
|
+
ls_result = await self.client.ls(cid)
|
1585
|
+
is_directory = ls_result.get("is_directory", False)
|
1586
|
+
result["is_directory"] = is_directory
|
1587
|
+
|
1588
|
+
# If it's a directory, recursively unpin all contained files first
|
1589
|
+
if is_directory:
|
1590
|
+
print(f"Detected directory: {cid}")
|
1591
|
+
links = []
|
1592
|
+
|
1593
|
+
# Extract all links from the directory listing
|
1594
|
+
if "Objects" in ls_result and len(ls_result["Objects"]) > 0:
|
1595
|
+
for obj in ls_result["Objects"]:
|
1596
|
+
if "Links" in obj:
|
1597
|
+
links.extend(obj["Links"])
|
1598
|
+
|
1599
|
+
child_files = []
|
1600
|
+
# Unpin each item in the directory
|
1601
|
+
for link in links:
|
1602
|
+
link_hash = link.get("Hash")
|
1603
|
+
link_name = link.get("Name", "unknown")
|
1604
|
+
if link_hash:
|
1605
|
+
child_files.append({"cid": link_hash, "name": link_name})
|
1606
|
+
try:
|
1607
|
+
# Recursively delete if it's a subdirectory
|
1608
|
+
link_type = link.get("Type")
|
1609
|
+
if (
|
1610
|
+
link_type == 1
|
1611
|
+
or str(link_type) == "1"
|
1612
|
+
or link_type == "dir"
|
1613
|
+
):
|
1614
|
+
# Recursive delete, but don't cancel from blockchain (we'll do that for parent)
|
1615
|
+
await self.delete_file(
|
1616
|
+
link_hash, cancel_from_blockchain=False
|
1617
|
+
)
|
1618
|
+
else:
|
1619
|
+
# Regular file unpin
|
1620
|
+
try:
|
1621
|
+
await self.client.unpin(link_hash)
|
1622
|
+
print(
|
1623
|
+
f"Unpinned file: {link_name} (CID: {link_hash})"
|
1624
|
+
)
|
1625
|
+
except Exception as unpin_error:
|
1626
|
+
# Just note the error but don't let it stop the whole process
|
1627
|
+
# This is common with IPFS servers that may return 500 errors for
|
1628
|
+
# unpinning content that was never explicitly pinned
|
1629
|
+
print(
|
1630
|
+
f"Note: Could not unpin {link_name}: {str(unpin_error).split('For more information')[0]}"
|
1631
|
+
)
|
1632
|
+
except Exception as e:
|
1633
|
+
print(
|
1634
|
+
f"Warning: Problem processing child item {link_name}: {str(e).split('For more information')[0]}"
|
1635
|
+
)
|
1532
1636
|
|
1637
|
+
# Record the child files that were processed
|
1638
|
+
result["child_files"] = child_files
|
1639
|
+
except Exception as e:
|
1640
|
+
print(f"Warning: Failed to check if CID is a directory: {e}")
|
1641
|
+
# Continue with regular file unpin
|
1642
|
+
|
1643
|
+
# Now unpin the main file/directory
|
1644
|
+
try:
|
1645
|
+
print(f"Unpinning from IPFS: {cid}")
|
1533
1646
|
unpin_result = await self.client.unpin(cid)
|
1534
1647
|
result["unpin_result"] = unpin_result
|
1648
|
+
result["success"] = True
|
1535
1649
|
print("Successfully unpinned from IPFS")
|
1536
1650
|
except Exception as e:
|
1537
|
-
|
1538
|
-
|
1651
|
+
# Handle 500 errors from IPFS server gracefully - they often occur
|
1652
|
+
# when the content wasn't explicitly pinned or was already unpinned
|
1653
|
+
error_str = str(e)
|
1654
|
+
if "500 Internal Server Error" in error_str:
|
1655
|
+
print(
|
1656
|
+
f"Note: IPFS server reported content may already be unpinned: {cid}"
|
1657
|
+
)
|
1658
|
+
result["unpin_result"] = {"Pins": [cid]} # Simulate successful unpin
|
1659
|
+
result["success"] = True
|
1660
|
+
else:
|
1661
|
+
print(
|
1662
|
+
f"Warning: Failed to unpin from IPFS: {error_str.split('For more information')[0]}"
|
1663
|
+
)
|
1664
|
+
result["success"] = False
|
1539
1665
|
|
1540
1666
|
# Then, if requested, cancel from blockchain
|
1541
1667
|
if cancel_from_blockchain:
|
1542
1668
|
try:
|
1543
|
-
# Create a substrate client
|
1544
|
-
print(f"DEBUG: Creating SubstrateClient for blockchain cancellation...")
|
1545
1669
|
substrate_client = SubstrateClient()
|
1546
|
-
|
1547
|
-
|
1548
|
-
|
1549
|
-
|
1550
|
-
|
1551
|
-
|
1552
|
-
|
1553
|
-
|
1554
|
-
# Check the return value - special cases for when blockchain cancellation isn't available
|
1555
|
-
if tx_hash == "no-blockchain-cancellation-available":
|
1556
|
-
print(
|
1557
|
-
"Blockchain cancellation not available, but IPFS unpinning was successful"
|
1558
|
-
)
|
1559
|
-
result["blockchain_result"] = {
|
1560
|
-
"status": "not_available",
|
1561
|
-
"message": "Blockchain cancellation not available, but IPFS unpinning was successful",
|
1562
|
-
}
|
1563
|
-
elif tx_hash.startswith("ipfs-unpinned-only"):
|
1564
|
-
error_msg = tx_hash.replace("ipfs-unpinned-only-", "")
|
1670
|
+
await substrate_client.cancel_storage_request(cid)
|
1671
|
+
print("Successfully cancelled storage from blockchain")
|
1672
|
+
result["blockchain_result"] = {"success": True}
|
1673
|
+
except Exception as e:
|
1674
|
+
# Handle the case where the CID is not in storage requests
|
1675
|
+
error_str = str(e)
|
1676
|
+
if "not found in storage requests" in error_str:
|
1565
1677
|
print(
|
1566
|
-
|
1678
|
+
"Note: Content was not found in blockchain storage requests (may already be deleted)"
|
1567
1679
|
)
|
1568
1680
|
result["blockchain_result"] = {
|
1569
|
-
"
|
1570
|
-
"
|
1571
|
-
"message": "IPFS unpinning successful, but blockchain cancellation failed",
|
1681
|
+
"success": True,
|
1682
|
+
"already_deleted": True,
|
1572
1683
|
}
|
1573
1684
|
else:
|
1574
|
-
|
1575
|
-
result["blockchain_result"] = {
|
1576
|
-
"transaction_hash": tx_hash,
|
1577
|
-
"status": "success",
|
1578
|
-
}
|
1579
|
-
print(f"Successfully canceled storage request from blockchain")
|
1580
|
-
print(
|
1581
|
-
f"DEBUG: Blockchain cancellation succeeded with transaction hash: {tx_hash}"
|
1582
|
-
)
|
1583
|
-
except Exception as e:
|
1584
|
-
print(f"Warning: Failed to cancel storage from blockchain: {e}")
|
1585
|
-
print(
|
1586
|
-
f"DEBUG: Blockchain cancellation exception: {type(e).__name__}: {str(e)}"
|
1587
|
-
)
|
1588
|
-
if hasattr(e, "__dict__"):
|
1589
|
-
print(f"DEBUG: Exception attributes: {e.__dict__}")
|
1590
|
-
result["blockchain_error"] = str(e)
|
1685
|
+
print(f"Warning: Error cancelling from blockchain: {error_str}")
|
1686
|
+
result["blockchain_result"] = {"success": False, "error": error_str}
|
1591
1687
|
|
1592
|
-
#
|
1688
|
+
# Update timing information
|
1593
1689
|
result["timing"]["end_time"] = time.time()
|
1594
1690
|
result["timing"]["duration_seconds"] = (
|
1595
1691
|
result["timing"]["end_time"] - result["timing"]["start_time"]
|
@@ -1602,7 +1698,7 @@ class IPFSClient:
|
|
1602
1698
|
metadata_cid: str,
|
1603
1699
|
cancel_from_blockchain: bool = True,
|
1604
1700
|
parallel_limit: int = 20,
|
1605
|
-
) ->
|
1701
|
+
) -> bool:
|
1606
1702
|
"""
|
1607
1703
|
Delete an erasure-coded file, including all its chunks in parallel.
|
1608
1704
|
|
@@ -1612,219 +1708,84 @@ class IPFSClient:
|
|
1612
1708
|
parallel_limit: Maximum number of concurrent deletion operations
|
1613
1709
|
|
1614
1710
|
Returns:
|
1615
|
-
|
1711
|
+
bool: True if the deletion was successful, False otherwise
|
1616
1712
|
"""
|
1617
|
-
result = {
|
1618
|
-
"metadata_cid": metadata_cid,
|
1619
|
-
"deleted_chunks": [],
|
1620
|
-
"failed_chunks": [],
|
1621
|
-
"blockchain_result": None,
|
1622
|
-
"timing": {
|
1623
|
-
"start_time": time.time(),
|
1624
|
-
"end_time": None,
|
1625
|
-
"duration_seconds": None,
|
1626
|
-
},
|
1627
|
-
}
|
1628
1713
|
|
1629
|
-
#
|
1630
|
-
|
1631
|
-
|
1714
|
+
# Try to download and process metadata file and chunks
|
1715
|
+
ipfs_failure = False
|
1716
|
+
metadata_error = False
|
1632
1717
|
|
1633
|
-
# First, get the metadata to find all chunks
|
1634
1718
|
try:
|
1635
|
-
|
1636
|
-
|
1637
|
-
|
1638
|
-
|
1639
|
-
|
1640
|
-
|
1641
|
-
|
1642
|
-
|
1643
|
-
|
1644
|
-
|
1645
|
-
|
1646
|
-
|
1647
|
-
|
1648
|
-
|
1649
|
-
|
1650
|
-
|
1651
|
-
|
1652
|
-
|
1653
|
-
|
1654
|
-
|
1719
|
+
# First download the metadata to get chunk CIDs
|
1720
|
+
try:
|
1721
|
+
metadata_result = await self.cat(metadata_cid)
|
1722
|
+
metadata_json = json.loads(metadata_result["content"].decode("utf-8"))
|
1723
|
+
chunks = metadata_json.get("chunks", [])
|
1724
|
+
except json.JSONDecodeError:
|
1725
|
+
# If we can't parse the metadata JSON, record the error but continue
|
1726
|
+
metadata_error = True
|
1727
|
+
# Continue with empty chunks so we can at least try to unpin the metadata file
|
1728
|
+
chunks = []
|
1729
|
+
except Exception:
|
1730
|
+
# Any other metadata error
|
1731
|
+
metadata_error = True
|
1732
|
+
chunks = []
|
1733
|
+
|
1734
|
+
# Extract all chunk CIDs
|
1735
|
+
chunk_cids = []
|
1736
|
+
for chunk in chunks:
|
1737
|
+
chunk_cid = chunk.get("cid", {})
|
1738
|
+
if isinstance(chunk_cid, dict) and "cid" in chunk_cid:
|
1739
|
+
chunk_cids.append(chunk_cid["cid"])
|
1740
|
+
elif isinstance(chunk_cid, str):
|
1741
|
+
chunk_cids.append(chunk_cid)
|
1655
1742
|
|
1656
1743
|
# Create a semaphore to limit concurrent operations
|
1657
|
-
|
1744
|
+
semaphore = asyncio.Semaphore(parallel_limit)
|
1658
1745
|
|
1659
|
-
# Define the chunk
|
1660
|
-
async def
|
1661
|
-
async with
|
1746
|
+
# Define the unpin task for each chunk with error handling
|
1747
|
+
async def unpin_chunk(cid):
|
1748
|
+
async with semaphore:
|
1662
1749
|
try:
|
1663
|
-
|
1664
|
-
|
1665
|
-
|
1666
|
-
# Record
|
1667
|
-
|
1668
|
-
|
1669
|
-
|
1670
|
-
|
1671
|
-
|
1672
|
-
|
1673
|
-
|
1674
|
-
|
1675
|
-
|
1676
|
-
|
1677
|
-
|
1678
|
-
|
1679
|
-
|
1680
|
-
|
1681
|
-
|
1682
|
-
|
1683
|
-
|
1684
|
-
|
1685
|
-
|
1686
|
-
|
1687
|
-
|
1688
|
-
"status": "not_available",
|
1689
|
-
"message": "Blockchain cancellation not available",
|
1690
|
-
}
|
1691
|
-
)
|
1692
|
-
elif tx_hash.startswith("ipfs-unpinned-only"):
|
1693
|
-
error_msg = tx_hash.replace(
|
1694
|
-
"ipfs-unpinned-only-", ""
|
1695
|
-
)
|
1696
|
-
result["blockchain_result"]["chunk_results"].append(
|
1697
|
-
{
|
1698
|
-
"cid": chunk_cid,
|
1699
|
-
"status": "failed",
|
1700
|
-
"error": error_msg,
|
1701
|
-
}
|
1702
|
-
)
|
1703
|
-
else:
|
1704
|
-
# Standard successful transaction
|
1705
|
-
result["blockchain_result"]["chunk_results"].append(
|
1706
|
-
{
|
1707
|
-
"cid": chunk_cid,
|
1708
|
-
"transaction_hash": tx_hash,
|
1709
|
-
"status": "success",
|
1710
|
-
}
|
1711
|
-
)
|
1712
|
-
except Exception as e:
|
1713
|
-
print(
|
1714
|
-
f"Warning: Failed to cancel blockchain storage for chunk {chunk_cid}: {e}"
|
1715
|
-
)
|
1716
|
-
|
1717
|
-
if "chunk_results" not in result["blockchain_result"]:
|
1718
|
-
result["blockchain_result"] = {}
|
1719
|
-
result["blockchain_result"]["chunk_results"] = []
|
1720
|
-
|
1721
|
-
result["blockchain_result"]["chunk_results"].append(
|
1722
|
-
{
|
1723
|
-
"cid": chunk_cid,
|
1724
|
-
"error": str(e),
|
1725
|
-
"status": "failed",
|
1726
|
-
}
|
1727
|
-
)
|
1728
|
-
|
1729
|
-
return True
|
1730
|
-
except Exception as e:
|
1731
|
-
error_msg = f"Failed to delete chunk {chunk_cid}: {e}"
|
1732
|
-
print(f"Warning: {error_msg}")
|
1733
|
-
|
1734
|
-
# Record failure
|
1735
|
-
async with failed_chunks_lock:
|
1736
|
-
result["failed_chunks"].append(
|
1737
|
-
{"cid": chunk_cid, "error": str(e)}
|
1738
|
-
)
|
1739
|
-
|
1740
|
-
return False
|
1741
|
-
|
1742
|
-
# Start deleting chunks in parallel
|
1743
|
-
print(
|
1744
|
-
f"Starting parallel deletion of {total_chunks} chunks with max {parallel_limit} concurrent operations"
|
1745
|
-
)
|
1746
|
-
delete_tasks = [delete_chunk(cid) for cid in chunks]
|
1747
|
-
await asyncio.gather(*delete_tasks)
|
1748
|
-
|
1749
|
-
# Delete the metadata file itself
|
1750
|
-
print(f"Unpinning metadata file: {metadata_cid}")
|
1751
|
-
response = await self.client.unpin(metadata_cid)
|
1752
|
-
|
1753
|
-
print(">>>", response)
|
1754
|
-
raise SystemExit
|
1755
|
-
|
1756
|
-
# Cancel metadata from blockchain if requested
|
1757
|
-
if cancel_from_blockchain:
|
1758
|
-
try:
|
1759
|
-
print(f"Canceling blockchain storage request for metadata file...")
|
1760
|
-
substrate_client = SubstrateClient()
|
1761
|
-
tx_hash = await substrate_client.cancel_storage_request(
|
1762
|
-
metadata_cid
|
1763
|
-
)
|
1764
|
-
|
1765
|
-
# Handle special return values from cancel_storage_request
|
1766
|
-
if tx_hash == "no-blockchain-cancellation-available":
|
1767
|
-
print(
|
1768
|
-
"Blockchain cancellation not available for metadata, but IPFS unpinning was successful"
|
1769
|
-
)
|
1770
|
-
result["blockchain_result"] = {
|
1771
|
-
"status": "not_available",
|
1772
|
-
"message": "Blockchain cancellation not available, but IPFS unpinning was successful",
|
1773
|
-
}
|
1774
|
-
elif tx_hash.startswith("ipfs-unpinned-only"):
|
1775
|
-
error_msg = tx_hash.replace("ipfs-unpinned-only-", "")
|
1776
|
-
print(
|
1777
|
-
f"IPFS unpinning successful, but blockchain cancellation failed for metadata: {error_msg}"
|
1778
|
-
)
|
1779
|
-
result["blockchain_result"] = {
|
1780
|
-
"status": "failed",
|
1781
|
-
"error": error_msg,
|
1782
|
-
"message": "IPFS unpinning successful, but blockchain cancellation failed",
|
1783
|
-
}
|
1784
|
-
else:
|
1785
|
-
# Standard successful transaction
|
1786
|
-
result["blockchain_result"] = {
|
1787
|
-
"metadata_transaction_hash": tx_hash,
|
1788
|
-
"status": "success",
|
1789
|
-
}
|
1790
|
-
print(
|
1791
|
-
f"Successfully canceled blockchain storage for metadata file"
|
1792
|
-
)
|
1793
|
-
except Exception as e:
|
1794
|
-
print(
|
1795
|
-
f"Warning: Failed to cancel blockchain storage for metadata file: {e}"
|
1796
|
-
)
|
1797
|
-
|
1798
|
-
if not result["blockchain_result"]:
|
1799
|
-
result["blockchain_result"] = {}
|
1800
|
-
|
1801
|
-
result["blockchain_result"]["metadata_error"] = str(e)
|
1802
|
-
result["blockchain_result"]["status"] = "failed"
|
1803
|
-
|
1804
|
-
# Calculate and record timing information
|
1805
|
-
end_time = time.time()
|
1806
|
-
duration = end_time - result["timing"]["start_time"]
|
1807
|
-
|
1808
|
-
result["timing"]["end_time"] = end_time
|
1809
|
-
result["timing"]["duration_seconds"] = duration
|
1810
|
-
|
1811
|
-
deleted_count = len(result["deleted_chunks"])
|
1812
|
-
failed_count = len(result["failed_chunks"])
|
1813
|
-
|
1814
|
-
print(f"Deletion complete in {duration:.2f} seconds!")
|
1815
|
-
print(f"Successfully deleted: {deleted_count}/{total_chunks} chunks")
|
1816
|
-
|
1817
|
-
if failed_count > 0:
|
1818
|
-
print(f"Failed to delete: {failed_count}/{total_chunks} chunks")
|
1750
|
+
await self.client.unpin(cid)
|
1751
|
+
return {"success": True, "cid": cid}
|
1752
|
+
except Exception:
|
1753
|
+
# Record failure but continue with other chunks
|
1754
|
+
return {"success": False, "cid": cid}
|
1755
|
+
|
1756
|
+
# Unpin all chunks in parallel
|
1757
|
+
if chunk_cids:
|
1758
|
+
unpin_tasks = [unpin_chunk(cid) for cid in chunk_cids]
|
1759
|
+
results = await asyncio.gather(*unpin_tasks)
|
1760
|
+
|
1761
|
+
# Count failures
|
1762
|
+
failures = [r for r in results if not r["success"]]
|
1763
|
+
if failures:
|
1764
|
+
ipfs_failure = True
|
1765
|
+
except Exception:
|
1766
|
+
# If we can't process chunks at all, record the failure
|
1767
|
+
ipfs_failure = True
|
1768
|
+
|
1769
|
+
# Unpin the metadata file itself, regardless of whether we could process chunks
|
1770
|
+
try:
|
1771
|
+
await self.client.unpin(metadata_cid)
|
1772
|
+
except Exception:
|
1773
|
+
# Record the failure but continue with blockchain cancellation
|
1774
|
+
ipfs_failure = True
|
1819
1775
|
|
1820
|
-
|
1821
|
-
|
1822
|
-
#
|
1823
|
-
|
1824
|
-
result["timing"]["duration_seconds"] = (
|
1825
|
-
result["timing"]["end_time"] - result["timing"]["start_time"]
|
1826
|
-
)
|
1776
|
+
# Handle blockchain cancellation if requested
|
1777
|
+
if cancel_from_blockchain:
|
1778
|
+
# Create a substrate client
|
1779
|
+
substrate_client = SubstrateClient()
|
1827
1780
|
|
1828
|
-
|
1829
|
-
|
1830
|
-
|
1781
|
+
# This will raise appropriate exceptions if it fails:
|
1782
|
+
# - HippiusAlreadyDeletedError if already deleted
|
1783
|
+
# - HippiusFailedSubstrateDelete if transaction fails
|
1784
|
+
# - Other exceptions for other failures
|
1785
|
+
await substrate_client.cancel_storage_request(metadata_cid)
|
1786
|
+
|
1787
|
+
# If we get here, either:
|
1788
|
+
# 1. Blockchain cancellation succeeded (if requested)
|
1789
|
+
# 2. We weren't doing blockchain cancellation
|
1790
|
+
# In either case, we report success
|
1791
|
+
return True
|