helppeer 0.1.0__tar.gz

helppeer-0.1.0/PKG-INFO

@@ -0,0 +1,94 @@
+Metadata-Version: 2.4
+Name: helppeer
+Version: 0.1.0
+Summary: Asynchronous ephemeral file transfer for AI model weights
+Author: JenD86
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/JenD86/help-peer
+Project-URL: Repository, https://github.com/JenD86/help-peer
+Project-URL: Issues, https://github.com/JenD86/help-peer/issues
+Keywords: file-transfer,ai,model-weights,safetensors,p2p,decentralized,ephemeral
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: cryptography>=41.0
+Requires-Dist: blake3>=0.3
+Requires-Dist: requests>=2.28
+Requires-Dist: reedsolo>=1.7
+Requires-Dist: hkdf>=0.0.3
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+
+# Help Peer Python SDK
+
+A Python SDK for asynchronously sharing large file dumps (AI model weights) via the Help Peer network.
+
+## Installation
+
+```bash
+pip install helppeer
+```
+
+## Quick Start
+
+```python
+import helppeer
+
+# Send a model directory
+code = helppeer.send("./my-model", name="Llama-3-70B")
+print(f"Transfer code: {code}")
+# Share this code with the recipient. Data lives for 24 hours.
+
+# Receive a transfer
+helppeer.receive("38-vortex-xenon", output_dir="./received")
+```
+
+## Configuration
+
+```python
+import helppeer
+
+# Use custom relay and storage nodes
+helppeer.configure(
+    relay="https://relay.helppeer.dev",
+    storage_nodes=["https://node1.helppeer.dev", "https://node2.helppeer.dev"],
+)
+
+# Or use environment variables:
+# HELPEER_RELAY_URL, HELPEER_STORAGE_NODES (comma-separated)
+```
+
+## Agent Integration
+
+```python
+import helppeer
+import json
+
+# Send and get structured result
+result = helppeer.send("./model-weights", name="my-model", return_details=True)
+# result = {"code": "38-vortex-xenon", "transfer_name": "my-model", "files": 4, "total_bytes": 1056}
+
+# Receive with progress callback
+def on_progress(file_path, segment, total_segments):
+    print(f"  {file_path}: {segment}/{total_segments} segments")
+
+helppeer.receive("38-vortex-xenon", output_dir="./received", progress=on_progress)
+```
+
+## CLI
+
+The SDK also installs a `helppeer` CLI:
+
+```bash
+helppeer send ./my-model --name "Llama-3-70B"
+helppeer receive 38-vortex-xenon --output ./received
+```

helppeer-0.1.0/README.md

@@ -0,0 +1,64 @@
+# Help Peer Python SDK
+
+A Python SDK for asynchronously sharing large file dumps (AI model weights) via the Help Peer network.
+
+## Installation
+
+```bash
+pip install helppeer
+```
+
+## Quick Start
+
+```python
+import helppeer
+
+# Send a model directory
+code = helppeer.send("./my-model", name="Llama-3-70B")
+print(f"Transfer code: {code}")
+# Share this code with the recipient. Data lives for 24 hours.
+
+# Receive a transfer
+helppeer.receive("38-vortex-xenon", output_dir="./received")
+```
+
+## Configuration
+
+```python
+import helppeer
+
+# Use custom relay and storage nodes
+helppeer.configure(
+    relay="https://relay.helppeer.dev",
+    storage_nodes=["https://node1.helppeer.dev", "https://node2.helppeer.dev"],
+)
+
+# Or use environment variables:
+# HELPEER_RELAY_URL, HELPEER_STORAGE_NODES (comma-separated)
+```
+
+## Agent Integration
+
+```python
+import helppeer
+import json
+
+# Send and get structured result
+result = helppeer.send("./model-weights", name="my-model", return_details=True)
+# result = {"code": "38-vortex-xenon", "transfer_name": "my-model", "files": 4, "total_bytes": 1056}
+
+# Receive with progress callback
+def on_progress(file_path, segment, total_segments):
+    print(f"  {file_path}: {segment}/{total_segments} segments")
+
+helppeer.receive("38-vortex-xenon", output_dir="./received", progress=on_progress)
+```
+
+## CLI
+
+The SDK also installs a `helppeer` CLI:
+
+```bash
+helppeer send ./my-model --name "Llama-3-70B"
+helppeer receive 38-vortex-xenon --output ./received
+```

helppeer-0.1.0/pyproject.toml

@@ -0,0 +1,48 @@
+[build-system]
+requires = ["setuptools>=64", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "helppeer"
+version = "0.1.0"
+description = "Asynchronous ephemeral file transfer for AI model weights"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.9"
+authors = [
+    {name = "JenD86"},
+]
+keywords = ["file-transfer", "ai", "model-weights", "safetensors", "p2p", "decentralized", "ephemeral"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "cryptography>=41.0",
+    "blake3>=0.3",
+    "requests>=2.28",
+    "reedsolo>=1.7",
+    "hkdf>=0.0.3",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=7.0"]
+
+[project.scripts]
+helppeer = "helppeer.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/JenD86/help-peer"
+Repository = "https://github.com/JenD86/help-peer"
+Issues = "https://github.com/JenD86/help-peer/issues"
+
+[tool.setuptools.packages.find]
+where = ["src"]

helppeer-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

helppeer-0.1.0/src/helppeer/__init__.py

@@ -0,0 +1,8 @@
+"""Help Peer — Asynchronous ephemeral file transfer for AI model weights."""
+
+from .client import send, receive
+from .config import configure, Config
+
+__version__ = "0.1.0"
+
+__all__ = ["send", "receive", "configure", "Config", "__version__"]

helppeer-0.1.0/src/helppeer/__main__.py

@@ -0,0 +1,2 @@
+[project.scripts]
+helppeer = "helppeer.cli:main"

helppeer-0.1.0/src/helppeer/cli.py

@@ -0,0 +1,68 @@
+"""Help Peer CLI entry point."""
+import sys
+import argparse
+from . import send, receive, configure
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="helppeer",
+        description="Asynchronous ephemeral file transfer for AI model weights",
+    )
+    parser.add_argument("--relay", default="http://127.0.0.1:7000", help="Relay server URL")
+    parser.add_argument("--nodes", default="http://127.0.0.1:7001",
+                        help="Storage node URLs (comma-separated)")
+
+    sub = parser.add_subcommands(dest="command", required=True)
+
+    send_cmd = sub.add_parser("send", help="Send a file or directory")
+    send_cmd.add_argument("path", help="Path to send")
+    send_cmd.add_argument("--name", default="untitled-transfer", help="Transfer name")
+
+    recv_cmd = sub.add_parser("receive", help="Receive a transfer")
+    recv_cmd.add_argument("code", help="Transfer code (e.g., 38-vortex-xenon)")
+    recv_cmd.add_argument("--output", default="./received", help="Output directory")
+
+    args = parser.parse_args()
+
+    # Configure from CLI args
+    nodes = [n.strip() for n in args.nodes.split(",") if n.strip()]
+    configure(relay=args.relay, storage_nodes=nodes)
+
+    if args.command == "send":
+        print(f"Help Peer — Sending: {args.path}")
+        print(f"Transfer name: {args.name}")
+        print(f"Relay: {args.relay}")
+        print(f"Storage nodes: {len(nodes)} (total)")
+        print()
+
+        code = send(args.path, name=args.name)
+        print()
+        print("✓ Upload complete!")
+        print()
+        print(f"  Transfer code: {code}")
+        print()
+        print("  Share this code with the recipient.")
+        print("  The data will be available for 24 hours.")
+
+    elif args.command == "receive":
+        print(f"Help Peer — Receiving with code: {args.code}")
+        print(f"Output directory: {args.output}")
+        print(f"Relay: {args.relay}")
+        print()
+
+        result = receive(args.code, output_dir=args.output)
+        print()
+        print("✓ Download complete!")
+        print()
+        print(f"  Transfer: {result['transfer_name']}")
+        print(f"  Files: {result['files']}")
+        print(f"  Total size: {result['total_bytes']} bytes")
+        print()
+        print("  File hashes (BLAKE3):")
+        for path, h in result["file_hashes"].items():
+            print(f"  {path} → {h}")
+
+
+if __name__ == "__main__":
+    main()

helppeer-0.1.0/src/helppeer/client.py

@@ -0,0 +1,278 @@
+"""Help Peer client — send and receive file transfers."""
+import os
+import random
+import requests
+from typing import Optional, Callable, List, Dict, Any
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from .config import get_config
+from . import crypto
+from . import erasure
+from .manifest import (
+    Manifest, ManifestFile, ManifestSegment, ManifestShard,
+    build_manifest, read_file_segment,
+)
+from .validator import ValidatorRegistry
+
+
+def send(
+    path: str,
+    name: str = "untitled-transfer",
+    *,
+    return_details: bool = False,
+) -> str | Dict[str, Any]:
+    """Send a file or directory. Returns the transfer code.
+
+    Args:
+        path: Path to the file or directory to send.
+        name: Human-readable name for the transfer.
+        return_details: If True, return a dict with full transfer details.
+
+    Returns:
+        Transfer code string (e.g., "38-vortex-xenon"), or a dict if return_details=True.
+    """
+    config = get_config()
+    code = _generate_code()
+    k_data, k_index = crypto.derive_keys(code)
+    r_hash = crypto.relay_hash(k_index)
+
+    if os.path.isfile(path):
+        # Single file: wrap in a temporary directory structure
+        manifest = _send_single_file(path, name, k_data, config)
+    else:
+        manifest = _send_directory(path, name, k_data, config)
+
+    # Encrypt and upload manifest
+    manifest_json = manifest.to_json()
+    encrypted_manifest = crypto.encrypt_segment(k_data, manifest_json)
+    _upload_manifest(config.relay_url, r_hash, encrypted_manifest)
+
+    if return_details:
+        return {
+            "code": code,
+            "transfer_name": name,
+            "files": len(manifest.files),
+            "total_bytes": manifest.total_bytes,
+        }
+    return code
+
+
+def receive(
+    code: str,
+    output_dir: str = "./received",
+    *,
+    progress: Optional[Callable[[str, int, int], None]] = None,
+) -> Dict[str, Any]:
+    """Receive a transfer using a code.
+
+    Args:
+        code: The transfer code (e.g., "38-vortex-xenon").
+        output_dir: Directory to save received files.
+        progress: Optional callback(file_path, segment_index, total_segments).
+
+    Returns:
+        Dict with transfer details: transfer_name, files, total_bytes, file_hashes.
+    """
+    config = get_config()
+    k_data, k_index = crypto.derive_keys(code)
+    r_hash = crypto.relay_hash(k_index)
+
+    # Download encrypted manifest from relay
+    encrypted_manifest = _download_manifest(config.relay_url, r_hash)
+    manifest_json = crypto.decrypt_segment(k_data, encrypted_manifest)
+    manifest = Manifest.from_json(manifest_json)
+
+    # Pre-allocate files
+    os.makedirs(output_dir, exist_ok=True)
+    for f in manifest.files:
+        file_path = os.path.join(output_dir, f.path)
+        os.makedirs(os.path.dirname(file_path) or ".", exist_ok=True)
+        with open(file_path, "wb") as fh:
+            fh.truncate(f.size)
+
+    # Download and reconstruct each file
+    registry = ValidatorRegistry()
+    file_hashes = {}
+
+    for f in manifest.files:
+        file_path = os.path.join(output_dir, f.path)
+        total_segments = len(f.segments)
+
+        for seg_idx, segment in enumerate(f.segments):
+            # Download shards
+            shard_data = _download_shards(segment)
+
+            # Reconstruct encrypted segment
+            encrypted = erasure.decode_segment(shard_data, segment.encrypted_size)
+
+            # Decrypt
+            plaintext = crypto.decrypt_segment(k_data, encrypted)
+
+            # Validate first segment
+            if seg_idx == 0:
+                validator = registry.validator_for(f.path)
+                validator.validate_first_segment(plaintext)
+
+            # Write to file at offset
+            offset = seg_idx * config.segment_size
+            with open(file_path, "r+b") as fh:
+                fh.seek(offset)
+                fh.write(plaintext)
+
+            if progress:
+                progress(f.path, seg_idx + 1, total_segments)
+
+        # Compute file hash
+        file_hashes[f.path] = _compute_file_hash(file_path)
+
+    return {
+        "transfer_name": manifest.transfer_name,
+        "files": len(manifest.files),
+        "total_bytes": manifest.total_bytes,
+        "file_hashes": file_hashes,
+    }
+
+
+def _send_directory(dir_path: str, name: str, k_data: bytes, config) -> Manifest:
+    """Send a directory."""
+    manifest = build_manifest(dir_path, name, config.segment_size)
+    manifest.erasure_data_shards = config.data_shards
+    manifest.erasure_parity_shards = config.parity_shards
+
+    for f in manifest.files:
+        file_path = os.path.join(dir_path, f.path)
+        for seg in f.segments:
+            seg_idx = int(seg.id.split("_")[1])
+            plaintext = read_file_segment(file_path, seg_idx, config.segment_size)
+            encrypted = crypto.encrypt_segment(k_data, plaintext)
+            shards = erasure.encode_segment(encrypted)
+
+            shard_infos = []
+            for shard_idx, shard in enumerate(shards):
+                h = crypto.content_hash(shard)
+                node = config.storage_nodes[shard_idx % len(config.storage_nodes)]
+                _upload_shard(node, h, shard)
+                shard_infos.append(ManifestShard(
+                    index=shard_idx, hash=h, node=node,
+                ))
+
+            seg.encrypted_size = len(encrypted)
+            seg.shards = shard_infos
+
+    return manifest
+
+
+def _send_single_file(file_path: str, name: str, k_data: bytes, config) -> Manifest:
+    """Send a single file by treating it as a one-file directory."""
+    dir_path = os.path.dirname(file_path) or "."
+    filename = os.path.basename(file_path)
+    manifest = build_manifest(dir_path, name, config.segment_size)
+    manifest.erasure_data_shards = config.data_shards
+    manifest.erasure_parity_shards = config.parity_shards
+
+    # Filter to only the requested file
+    manifest.files = [f for f in manifest.files if f.path == filename]
+    manifest.total_bytes = sum(f.size for f in manifest.files)
+
+    for f in manifest.files:
+        full_path = os.path.join(dir_path, f.path)
+        for seg in f.segments:
+            seg_idx = int(seg.id.split("_")[1])
+            plaintext = read_file_segment(full_path, seg_idx, config.segment_size)
+            encrypted = crypto.encrypt_segment(k_data, plaintext)
+            shards = erasure.encode_segment(encrypted)
+
+            shard_infos = []
+            for shard_idx, shard in enumerate(shards):
+                h = crypto.content_hash(shard)
+                node = config.storage_nodes[shard_idx % len(config.storage_nodes)]
+                _upload_shard(node, h, shard)
+                shard_infos.append(ManifestShard(
+                    index=shard_idx, hash=h, node=node,
+                ))
+
+            seg.encrypted_size = len(encrypted)
+            seg.shards = shard_infos
+
+    return manifest
+
+
+def _upload_shard(node_url: str, hash_hex: str, data: bytes) -> None:
+    """Upload a shard to a storage node."""
+    url = f"{node_url}/shard/{hash_hex}"
+    resp = requests.put(url, data=data, headers={"Content-Type": "application/octet-stream"})
+    if not resp.status_code in (200, 201):
+        raise RuntimeError(f"shard upload to {node_url} returned {resp.status_code}")
+
+
+def _upload_manifest(relay_url: str, r_hash: str, data: bytes) -> None:
+    """Upload encrypted manifest to relay."""
+    url = f"{relay_url}/manifest/{r_hash}"
+    resp = requests.put(url, data=data, headers={"Content-Type": "application/octet-stream"})
+    if not resp.status_code in (200, 201):
+        raise RuntimeError(f"manifest upload returned {resp.status_code}")
+
+
+def _download_manifest(relay_url: str, r_hash: str) -> bytes:
+    """Download encrypted manifest from relay (one-time retrieval)."""
+    url = f"{relay_url}/manifest/{r_hash}"
+    resp = requests.get(url)
+    if resp.status_code != 200:
+        raise RuntimeError(f"manifest download returned {resp.status_code}")
+    return resp.content
+
+
+def _download_shards(segment: ManifestSegment) -> List[Optional[bytes]]:
+    """Download all shards for a segment concurrently."""
+    config = get_config()
+    total_shards = config.data_shards + config.parity_shards
+    shards: List[Optional[bytes]] = [None] * total_shards
+
+    def fetch_shard(shard: ManifestShard) -> tuple[int, Optional[bytes]]:
+        url = f"{shard.node}/shard/{shard.hash}"
+        try:
+            resp = requests.get(url)
+            if resp.status_code == 200:
+                return shard.index, resp.content
+        except Exception:
+            pass
+        return shard.index, None
+
+    with ThreadPoolExecutor(max_workers=12) as executor:
+        futures = [executor.submit(fetch_shard, s) for s in segment.shards]
+        for future in as_completed(futures):
+            idx, data = future.result()
+            if data is not None:
+                shards[idx] = data
+
+    return shards
+
+
+def _compute_file_hash(filepath: str) -> str:
+    """Compute BLAKE3 hash of a file."""
+    import blake3
+    h = blake3.blake3()
+    with open(filepath, "rb") as f:
+        while True:
+            chunk = f.read(1024 * 1024)
+            if not chunk:
+                break
+            h.update(chunk)
+    return h.hexdigest()
+
+
+_WORDS = [
+    "orbit", "velvet", "guitar", "battery", "candle", "dragon", "ember",
+    "falcon", "galaxy", "harbor", "iris", "jungle", "kettle", "lantern",
+    "meadow", "nebula", "ocean", "phoenix", "quartz", "raven", "silver",
+    "thunder", "umbra", "vortex", "willow", "xenon", "yonder", "zephyr",
+    "amber", "breeze",
+]
+
+
+def _generate_code() -> str:
+    """Generate a human-readable transfer code: number-word-word."""
+    num = random.randint(1, 99)
+    w1 = random.choice(_WORDS)
+    w2 = random.choice(_WORDS)
+    return f"{num}-{w1}-{w2}"

helppeer-0.1.0/src/helppeer/config.py

@@ -0,0 +1,45 @@
+"""Configuration for Help Peer SDK."""
+import os
+from dataclasses import dataclass, field
+from typing import List
+
+
+@dataclass
+class Config:
+    """SDK configuration. Modify via helppeer.configure()."""
+    relay_url: str = os.environ.get("HELPEER_RELAY_URL", "http://127.0.0.1:7000")
+    storage_nodes: List[str] = field(default_factory=lambda: [
+        n.strip() for n in os.environ.get(
+            "HELPEER_STORAGE_NODES", "http://127.0.0.1:7001"
+        ).split(",") if n.strip()
+    ])
+    segment_size: int = 64 * 1024 * 1024  # 64 MB
+    data_shards: int = 8
+    parity_shards: int = 4
+
+
+_default_config = Config()
+
+
+def get_config() -> Config:
+    return _default_config
+
+
+def configure(
+    relay: str = None,
+    storage_nodes: List[str] = None,
+    segment_size: int = None,
+    data_shards: int = None,
+    parity_shards: int = None,
+):
+    """Configure the SDK. Only provided values are updated."""
+    if relay is not None:
+        _default_config.relay_url = relay
+    if storage_nodes is not None:
+        _default_config.storage_nodes = storage_nodes
+    if segment_size is not None:
+        _default_config.segment_size = segment_size
+    if data_shards is not None:
+        _default_config.data_shards = data_shards
+    if parity_shards is not None:
+        _default_config.parity_shards = parity_shards

helppeer-0.1.0/src/helppeer/crypto.py

@@ -0,0 +1,59 @@
+"""Cryptographic operations for Help Peer."""
+import os
+import struct
+import hmac
+import hashlib
+import hkdf
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+NONCE_SIZE = 12
+KEY_SIZE = 32
+
+
+def derive_keys(code: str) -> tuple[bytes, bytes]:
+    """Derive K_data and K_index from a transfer code using HKDF-SHA256."""
+    ikm = code.encode("utf-8")
+    k_data = hkdf.Hkdf(None, ikm, hashlib.sha256).expand(b"help-peer-data-key", KEY_SIZE)
+    k_index = hkdf.Hkdf(None, ikm, hashlib.sha256).expand(b"help-peer-index-key", KEY_SIZE)
+    return k_data, k_index
+
+
+def relay_hash(k_index: bytes) -> str:
+    """Compute the relay hash: hex(BLAKE3(K_index))."""
+    import blake3
+    return blake3.blake3(k_index).hexdigest()
+
+
+def encrypt_segment(k_data: bytes, plaintext: bytes) -> bytes:
+    """Encrypt a segment with AES-256-GCM. Returns nonce || ciphertext."""
+    nonce = os.urandom(NONCE_SIZE)
+    aesgcm = AESGCM(k_data)
+    ciphertext = aesgcm.encrypt(nonce, plaintext, None)
+    return nonce + ciphertext
+
+
+def decrypt_segment(k_data: bytes, encrypted: bytes) -> bytes:
+    """Decrypt a segment (nonce || ciphertext) with AES-256-GCM."""
+    if len(encrypted) < NONCE_SIZE:
+        raise ValueError("encrypted data too short")
+    nonce = encrypted[:NONCE_SIZE]
+    ciphertext = encrypted[NONCE_SIZE:]
+    aesgcm = AESGCM(k_data)
+    return aesgcm.decrypt(nonce, ciphertext, None)
+
+
+def content_hash(data: bytes) -> str:
+    """Compute BLAKE3 hash of data, return hex string."""
+    import blake3
+    return blake3.blake3(data).hexdigest()
+
+
+def shard_hmac(k_data: bytes, shard: bytes) -> bytes:
+    """Compute HMAC-SHA256 of shard data."""
+    return hmac.new(k_data, shard, hashlib.sha256).digest()
+
+
+def verify_shard_hmac(k_data: bytes, shard: bytes, expected: bytes) -> bool:
+    """Verify shard HMAC."""
+    actual = shard_hmac(k_data, shard)
+    return hmac.compare_digest(actual, expected)

helppeer-0.1.0/src/helppeer/erasure.py

@@ -0,0 +1,130 @@
+"""Reed-Solomon erasure coding for Help Peer."""
+from reedsolo import RSCodec
+
+from .config import get_config
+
+
+def encode_segment(data: bytes) -> list[bytes]:
+    """Split an encrypted segment into (8 data + 4 parity) shards."""
+    config = get_config()
+    return _encode_interleaved(data, config.data_shards, config.parity_shards)
+
+
+def _encode_interleaved(data: bytes, data_shards: int, parity_shards: int) -> list[bytes]:
+    """Encode data using interleaved Reed-Solomon: split into data_shards pieces,
+    compute parity_shards parity pieces, return data_shards + parity_shards shards."""
+    shard_size = (len(data) + data_shards - 1) // data_shards
+    # Pad data to fill all data shards
+    padded = data + b"\x00" * (shard_size * data_shards - len(data))
+
+    # Split into data shards
+    data_shards_list = [
+        padded[i * shard_size:(i + 1) * shard_size]
+        for i in range(data_shards)
+    ]
+
+    # For each byte position, compute parity across data_shards bytes
+    rs = RSCodec(parity_shards, nsize=data_shards + parity_shards)
+
+    parity_shards_list = [bytearray(shard_size) for _ in range(parity_shards)]
+
+    for pos in range(shard_size):
+        # Gather one byte from each data shard
+        data_bytes = bytes(data_shards_list[i][pos] for i in range(data_shards))
+        # Encode: produces data_bytes + parity_bytes
+        encoded = rs.encode(data_bytes)
+        # Extract parity bytes
+        for j in range(parity_shards):
+            parity_shards_list[j][pos] = encoded[data_shards + j]
+
+    return data_shards_list + [bytes(p) for p in parity_shards_list]
+
+
+def decode_segment(shards: list[bytes | None], original_len: int) -> bytes:
+    """Reconstruct the original segment from at least data_shards of total_shards shards.
+    Missing shards should be None."""
+    config = get_config()
+    data_shards = config.data_shards
+    parity_shards = config.parity_shards
+    total_shards = data_shards + parity_shards
+
+    if len(shards) != total_shards:
+        raise ValueError(f"expected {total_shards} shards, got {len(shards)}")
+
+    # Find shard size from available shards
+    shard_size = None
+    for s in shards:
+        if s is not None:
+            shard_size = len(s)
+            break
+
+    if shard_size is None:
+        raise ValueError("no shards available")
+
+    # Count available shards
+    available = sum(1 for s in shards if s is not None)
+    if available < data_shards:
+        raise ValueError(f"insufficient shards: have {available}, need {data_shards}")
+
+    rs = RSCodec(parity_shards, nsize=data_shards + parity_shards)
+
+    # For each byte position, reconstruct missing bytes
+    data_shards_list = [None] * data_shards
+    parity_shards_list = [None] * parity_shards
+
+    for i in range(total_shards):
+        if shards[i] is not None:
+            if i < data_shards:
+                data_shards_list[i] = shards[i]
+            else:
+                parity_shards_list[i - data_shards] = shards[i]
+
+    result_shards = [bytearray(shard_size) for _ in range(data_shards)]
+
+    for pos in range(shard_size):
+        # Build the received codeword for this byte position
+        received = [None] * total_shards
+        for i in range(data_shards):
+            if data_shards_list[i] is not None:
+                received[i] = data_shards_list[i][pos]
+        for j in range(parity_shards):
+            if parity_shards_list[j] is not None:
+                received[data_shards + j] = parity_shards_list[j][pos]
+
+        # Convert to list with 0 for missing (erasure positions)
+        codeword = []
+        erasure_pos = []
+        for i in range(total_shards):
+            if received[i] is not None:
+                codeword.append(received[i])
+            else:
+                codeword.append(0)
+                erasure_pos.append(i)
+
+        if erasure_pos:
+            # Decode with erasures
+            try:
+                decoded = rs.decode(bytearray(codeword), erase_pos=erasure_pos)
+                # reedsolo returns (msg, full_corrected_codeword) when there are erasures
+                if isinstance(decoded, tuple):
+                    full_codeword = decoded[1]
+                else:
+                    full_codeword = decoded
+                # Extract corrected data bytes
+                for i in range(data_shards):
+                    result_shards[i][pos] = full_codeword[i]
+            except Exception:
+                # If decode fails, use available data bytes directly
+                for i in range(data_shards):
+                    if data_shards_list[i] is not None:
+                        result_shards[i][pos] = data_shards_list[i][pos]
+                    else:
+                        result_shards[i][pos] = 0
+        else:
+            # No erasures, just copy data bytes
+            for i in range(data_shards):
+                result_shards[i][pos] = codeword[i]
+
+    # Concatenate data shards and trim to original length
+    result = b"".join(bytes(s) for s in result_shards)
+    return result[:original_len]

helppeer-0.1.0/src/helppeer/manifest.py

@@ -0,0 +1,139 @@
+"""Manifest building and parsing for Help Peer."""
+import os
+import json
+from dataclasses import dataclass, field, asdict
+from typing import List, Optional
+
+
+@dataclass
+class ManifestShard:
+    index: int
+    hash: str
+    node: str
+
+
+@dataclass
+class ManifestSegment:
+    id: str
+    original_size: int
+    encrypted_size: int
+    shards: List[ManifestShard] = field(default_factory=list)
+
+
+@dataclass
+class ManifestFile:
+    path: str
+    size: int
+    segments: List[ManifestSegment] = field(default_factory=list)
+
+
+@dataclass
+class Manifest:
+    version: int = 1
+    transfer_name: str = ""
+    total_bytes: int = 0
+    segment_size: int = 67108864
+    erasure_data_shards: int = 8
+    erasure_parity_shards: int = 4
+    files: List[ManifestFile] = field(default_factory=list)
+
+    def to_json(self) -> bytes:
+        return json.dumps({
+            "version": self.version,
+            "transfer_name": self.transfer_name,
+            "total_bytes": self.total_bytes,
+            "segment_size": self.segment_size,
+            "erasure_data_shards": self.erasure_data_shards,
+            "erasure_parity_shards": self.erasure_parity_shards,
+            "files": [
+                {
+                    "path": f.path,
+                    "size": f.size,
+                    "segments": [
+                        {
+                            "id": s.id,
+                            "original_size": s.original_size,
+                            "encrypted_size": s.encrypted_size,
+                            "shards": [
+                                {"index": sh.index, "hash": sh.hash, "node": sh.node}
+                                for sh in s.shards
+                            ],
+                        }
+                        for s in f.segments
+                    ],
+                }
+                for f in self.files
+            ],
+        }).encode("utf-8")
+
+    @classmethod
+    def from_json(cls, data: bytes) -> "Manifest":
+        d = json.loads(data)
+        manifest = cls(
+            version=d["version"],
+            transfer_name=d["transfer_name"],
+            total_bytes=d["total_bytes"],
+            segment_size=d["segment_size"],
+            erasure_data_shards=d["erasure_data_shards"],
+            erasure_parity_shards=d["erasure_parity_shards"],
+        )
+        for f in d["files"]:
+            mf = ManifestFile(path=f["path"], size=f["size"])
+            for s in f["segments"]:
+                ms = ManifestSegment(
+                    id=s["id"],
+                    original_size=s["original_size"],
+                    encrypted_size=s["encrypted_size"],
+                )
+                for sh in s["shards"]:
+                    ms.shards.append(ManifestShard(
+                        index=sh["index"], hash=sh["hash"], node=sh["node"]
+                    ))
+                mf.segments.append(ms)
+            manifest.files.append(mf)
+        return manifest
+
+
+def build_manifest(dir_path: str, transfer_name: str, segment_size: int) -> Manifest:
+    """Build a manifest by crawling a directory."""
+    manifest = Manifest(
+        transfer_name=transfer_name,
+        segment_size=segment_size,
+    )
+
+    for root, dirs, files in os.walk(dir_path):
+        dirs.sort()
+        for filename in sorted(files):
+            filepath = os.path.join(root, filename)
+            rel_path = os.path.relpath(filepath, dir_path)
+
+            size = os.path.getsize(filepath)
+            manifest.total_bytes += size
+
+            num_segments = 0
+            if size > 0:
+                num_segments = (size + segment_size - 1) // segment_size
+
+            segments = []
+            for i in range(num_segments):
+                seg_size = min(segment_size, size - i * segment_size)
+                segments.append(ManifestSegment(
+                    id=f"seg_{i:06d}",
+                    original_size=seg_size,
+                    encrypted_size=0,
+                ))
+
+            manifest.files.append(ManifestFile(
+                path=rel_path,
+                size=size,
+                segments=segments,
+            ))
+
+    return manifest
+
+
+def read_file_segment(filepath: str, segment_index: int, segment_size: int) -> bytes:
+    """Read a specific segment from a file."""
+    with open(filepath, "rb") as f:
+        f.seek(segment_index * segment_size)
+        return f.read(segment_size)

helppeer-0.1.0/src/helppeer/validator.py

@@ -0,0 +1,66 @@
+"""Pluggable file validators for Help Peer."""
+import json
+from typing import Protocol
+
+
+class ValidationError(Exception):
+    pass
+
+
+class FileValidator(Protocol):
+    def file_extensions(self) -> list[str]: ...
+    def validate_first_segment(self, data: bytes) -> None: ...
+
+
+class SafetensorsValidator:
+    """Validates safetensors files by parsing the JSON header."""
+
+    def file_extensions(self) -> list[str]:
+        return ["safetensors"]
+
+    def validate_first_segment(self, data: bytes) -> None:
+        if len(data) < 8:
+            raise ValidationError("file too small to be a valid safetensors file")
+
+        header_len = int.from_bytes(data[:8], "little")
+
+        if header_len > 100_000_000:
+            raise ValidationError(f"safetensors header length {header_len} is unreasonably large")
+
+        if len(data) < 8 + header_len:
+            raise ValidationError("safetensors header extends beyond first segment")
+
+        header_json = data[8:8 + header_len]
+        try:
+            parsed = json.loads(header_json)
+        except json.JSONDecodeError as e:
+            raise ValidationError(f"safetensors header is not valid JSON: {e}")
+
+        if not isinstance(parsed, dict):
+            raise ValidationError("safetensors header is not a JSON object")
+
+        if len(parsed) == 0:
+            raise ValidationError("safetensors header is empty")
+
+
+class GenericValidator:
+    """Passthrough validator that accepts all data."""
+
+    def file_extensions(self) -> list[str]:
+        return []
+
+    def validate_first_segment(self, data: bytes) -> None:
+        pass
+
+
+class ValidatorRegistry:
+    def __init__(self):
+        self.validators: list[FileValidator] = [SafetensorsValidator()]
+        self.fallback: FileValidator = GenericValidator()
+
+    def validator_for(self, path: str) -> FileValidator:
+        ext = path.rsplit(".", 1)[-1].lower() if "." in path else ""
+        for v in self.validators:
+            if ext in v.file_extensions():
+                return v
+        return self.fallback

helppeer-0.1.0/src/helppeer.egg-info/PKG-INFO

@@ -0,0 +1,94 @@
+Metadata-Version: 2.4
+Name: helppeer
+Version: 0.1.0
+Summary: Asynchronous ephemeral file transfer for AI model weights
+Author: JenD86
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/JenD86/help-peer
+Project-URL: Repository, https://github.com/JenD86/help-peer
+Project-URL: Issues, https://github.com/JenD86/help-peer/issues
+Keywords: file-transfer,ai,model-weights,safetensors,p2p,decentralized,ephemeral
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: cryptography>=41.0
+Requires-Dist: blake3>=0.3
+Requires-Dist: requests>=2.28
+Requires-Dist: reedsolo>=1.7
+Requires-Dist: hkdf>=0.0.3
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+
+# Help Peer Python SDK
+
+A Python SDK for asynchronously sharing large file dumps (AI model weights) via the Help Peer network.
+
+## Installation
+
+```bash
+pip install helppeer
+```
+
+## Quick Start
+
+```python
+import helppeer
+
+# Send a model directory
+code = helppeer.send("./my-model", name="Llama-3-70B")
+print(f"Transfer code: {code}")
+# Share this code with the recipient. Data lives for 24 hours.
+
+# Receive a transfer
+helppeer.receive("38-vortex-xenon", output_dir="./received")
+```
+
+## Configuration
+
+```python
+import helppeer
+
+# Use custom relay and storage nodes
+helppeer.configure(
+    relay="https://relay.helppeer.dev",
+    storage_nodes=["https://node1.helppeer.dev", "https://node2.helppeer.dev"],
+)
+
+# Or use environment variables:
+# HELPEER_RELAY_URL, HELPEER_STORAGE_NODES (comma-separated)
+```
+
+## Agent Integration
+
+```python
+import helppeer
+import json
+
+# Send and get structured result
+result = helppeer.send("./model-weights", name="my-model", return_details=True)
+# result = {"code": "38-vortex-xenon", "transfer_name": "my-model", "files": 4, "total_bytes": 1056}
+
+# Receive with progress callback
+def on_progress(file_path, segment, total_segments):
+    print(f"  {file_path}: {segment}/{total_segments} segments")
+
+helppeer.receive("38-vortex-xenon", output_dir="./received", progress=on_progress)
+```
+
+## CLI
+
+The SDK also installs a `helppeer` CLI:
+
+```bash
+helppeer send ./my-model --name "Llama-3-70B"
+helppeer receive 38-vortex-xenon --output ./received
+```

helppeer-0.1.0/src/helppeer.egg-info/SOURCES.txt

@@ -0,0 +1,18 @@
+README.md
+pyproject.toml
+src/helppeer/__init__.py
+src/helppeer/__main__.py
+src/helppeer/cli.py
+src/helppeer/client.py
+src/helppeer/config.py
+src/helppeer/crypto.py
+src/helppeer/erasure.py
+src/helppeer/manifest.py
+src/helppeer/validator.py
+src/helppeer.egg-info/PKG-INFO
+src/helppeer.egg-info/SOURCES.txt
+src/helppeer.egg-info/dependency_links.txt
+src/helppeer.egg-info/entry_points.txt
+src/helppeer.egg-info/requires.txt
+src/helppeer.egg-info/top_level.txt
+tests/test_sdk.py

helppeer-0.1.0/src/helppeer.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

helppeer-0.1.0/src/helppeer.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+helppeer = helppeer.cli:main

helppeer-0.1.0/src/helppeer.egg-info/requires.txt

@@ -0,0 +1,8 @@
+cryptography>=41.0
+blake3>=0.3
+requests>=2.28
+reedsolo>=1.7
+hkdf>=0.0.3
+
+[dev]
+pytest>=7.0

helppeer-0.1.0/src/helppeer.egg-info/top_level.txt

@@ -0,0 +1 @@
+helppeer

helppeer-0.1.0/tests/test_sdk.py

@@ -0,0 +1,160 @@
+"""Tests for Help Peer SDK."""
+import os
+import json
+import struct
+import tempfile
+import shutil
+import pytest
+
+from helppeer import crypto, erasure, manifest, validator
+from helppeer.config import configure, get_config
+
+
+class TestCrypto:
+    def test_encrypt_decrypt_roundtrip(self):
+        k_data = b"\x00" * 32
+        plaintext = b"Hello, Help Peer!"
+        encrypted = crypto.encrypt_segment(k_data, plaintext)
+        decrypted = crypto.decrypt_segment(k_data, encrypted)
+        assert plaintext == decrypted
+
+    def test_key_derivation_consistency(self):
+        k1_data, k1_index = crypto.derive_keys("7-orbit-velvet")
+        k2_data, k2_index = crypto.derive_keys("7-orbit-velvet")
+        assert k1_data == k2_data
+        assert k1_index == k2_index
+
+        k3_data, _ = crypto.derive_keys("8-orbit-velvet")
+        assert k1_data != k3_data
+
+    def test_relay_hash_consistency(self):
+        _, k_index = crypto.derive_keys("7-orbit-velvet")
+        h1 = crypto.relay_hash(k_index)
+        h2 = crypto.relay_hash(k_index)
+        assert h1 == h2
+        assert len(h1) == 64  # 32 bytes hex
+
+    def test_hmac_verification(self):
+        k_data = b"\x00" * 32
+        shard = b"some shard data"
+        h = crypto.shard_hmac(k_data, shard)
+        assert crypto.verify_shard_hmac(k_data, shard, h)
+        assert not crypto.verify_shard_hmac(k_data, b"tampered", h)
+
+
+class TestErasure:
+    def test_encode_decode_roundtrip(self):
+        data = b"Hello, Help Peer! This is a test segment for erasure coding."
+        shards = erasure.encode_segment(data)
+        assert len(shards) == 12
+
+        shard_opts = list(shards)
+        reconstructed = erasure.decode_segment(shard_opts, len(data))
+        assert data == reconstructed
+
+    def test_decode_with_missing_parity(self):
+        data = b"Test data for erasure coding with missing shards!"
+        shards = erasure.encode_segment(data)
+
+        # Drop 4 parity shards
+        shard_opts = list(shards)
+        for i in range(8, 12):
+            shard_opts[i] = None
+        reconstructed = erasure.decode_segment(shard_opts, len(data))
+        assert data == reconstructed
+
+    def test_decode_with_missing_data_shards(self):
+        data = b"Test data for erasure coding with some data shards missing!"
+        shards = erasure.encode_segment(data)
+
+        # Drop 2 data + 2 parity
+        shard_opts = list(shards)
+        shard_opts[1] = None
+        shard_opts[3] = None
+        shard_opts[9] = None
+        shard_opts[11] = None
+        reconstructed = erasure.decode_segment(shard_opts, len(data))
+        assert data == reconstructed
+
+    def test_decode_insufficient_shards(self):
+        data = b"Too many missing shards!"
+        shards = erasure.encode_segment(data)
+
+        shard_opts = list(shards)
+        for i in range(5):
+            shard_opts[i] = None
+
+        with pytest.raises(ValueError, match="insufficient"):
+            erasure.decode_segment(shard_opts, len(data))
+
+
+class TestManifest:
+    def test_build_from_dir(self, tmp_path):
+        (tmp_path / "config.json").write_text("{}")
+        (tmp_path / "model.safetensors").write_text("weights")
+
+        m = manifest.build_manifest(str(tmp_path), "test-transfer", 67108864)
+        assert m.transfer_name == "test-transfer"
+        assert m.version == 1
+        assert len(m.files) == 2
+        assert m.total_bytes > 0
+
+    def test_json_roundtrip(self):
+        m = manifest.Manifest(
+            transfer_name="test",
+            total_bytes=100,
+            segment_size=67108864,
+            files=[
+                manifest.ManifestFile(
+                    path="test.txt",
+                    size=100,
+                    segments=[
+                        manifest.ManifestSegment(
+                            id="seg_000000",
+                            original_size=100,
+                            encrypted_size=0,
+                        )
+                    ],
+                )
+            ],
+        )
+        json_bytes = m.to_json()
+        restored = manifest.Manifest.from_json(json_bytes)
+        assert restored.transfer_name == "test"
+        assert len(restored.files) == 1
+        assert restored.files[0].path == "test.txt"
+
+
+class TestValidator:
+    def test_safetensors_valid(self):
+        header = json.dumps({
+            "__metadata__": {"format": "pt"},
+            "weight1": {"dtype": "F32", "shape": [10, 10], "data_offsets": [0, 400]},
+        }).encode()
+        data = struct.pack("<Q", len(header)) + header
+        v = validator.SafetensorsValidator()
+        v.validate_first_segment(data)  # should not raise
+
+    def test_safetensors_invalid_json(self):
+        data = struct.pack("<Q", 5) + b"NOT{JSON"
+        v = validator.SafetensorsValidator()
+        with pytest.raises(validator.ValidationError):
+            v.validate_first_segment(data)
+
+    def test_safetensors_too_small(self):
+        v = validator.SafetensorsValidator()
+        with pytest.raises(validator.ValidationError):
+            v.validate_first_segment(b"tiny")
+
+    def test_generic_accepts_all(self):
+        v = validator.GenericValidator()
+        v.validate_first_segment(b"anything")
+        v.validate_first_segment(b"")
+
+    def test_registry_lookup(self):
+        reg = validator.ValidatorRegistry()
+        v = reg.validator_for("model.safetensors")
+        assert "safetensors" in v.file_extensions()
+
+        v = reg.validator_for("config.json")
+        assert len(v.file_extensions()) == 0  # fallback