firecloud-devnet 0.1.0__py3-none-any.whl

firecloud/crypto.py

@@ -0,0 +1,269 @@
+"""FireCloud cryptographic engine.
+
+Provides chunk-level authenticated encryption (XChaCha20-Poly1305),
+convergent chunk addressing (HMAC-SHA-256), integrity verification
+(SHA-256), passphrase-protected keystore (scrypt + AES-256-GCM),
+and HKDF-based sub-key derivation.
+"""
+
+import hashlib
+import hmac
+
+from Crypto.Cipher import AES, ChaCha20_Poly1305
+from Crypto.Random import get_random_bytes
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+from cryptography.hazmat.primitives.kdf.scrypt import Scrypt
+
+from firecloud.exceptions import ChunkCorruptError, NetworkKeyError
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+_XCHACHA_NONCE_LEN = 24  # XChaCha20-Poly1305 nonce
+_AES_GCM_NONCE_LEN = 12  # AES-256-GCM nonce
+_SCRYPT_SALT_LEN = 16
+_KEY_LEN = 32  # 256-bit keys throughout
+
+# Scrypt cost parameters (interactive-grade, fast for dev/test)
+_SCRYPT_N = 2**14
+_SCRYPT_R = 8
+_SCRYPT_P = 1
+
+# ---------------------------------------------------------------------------
+# Chunk encryption / decryption
+# ---------------------------------------------------------------------------
+
+
+def encrypt_chunk(plaintext: bytes, key: bytes) -> bytes:
+    """Encrypt *plaintext* with XChaCha20-Poly1305.
+
+    Args:
+        plaintext: Arbitrary-length data (may be empty).
+        key: 32-byte symmetric key.
+
+    Returns:
+        ``nonce (24 B) || ciphertext || auth_tag (16 B)``
+    """
+    nonce = get_random_bytes(_XCHACHA_NONCE_LEN)
+    cipher = ChaCha20_Poly1305.new(key=key, nonce=nonce)
+    ciphertext, tag = cipher.encrypt_and_digest(plaintext)
+    return nonce + ciphertext + tag
+
+
+def decrypt_chunk(encrypted: bytes, key: bytes) -> bytes:
+    """Decrypt data produced by :func:`encrypt_chunk`.
+
+    Args:
+        encrypted: ``nonce (24 B) || ciphertext || auth_tag (16 B)``
+        key: 32-byte symmetric key (must match encryption key).
+
+    Returns:
+        The original plaintext bytes.
+
+    Raises:
+        ChunkCorruptError: Authentication tag verification failed
+            (wrong key, truncated data, or tampered ciphertext).
+    """
+    if len(encrypted) < _XCHACHA_NONCE_LEN + 16:
+        raise ChunkCorruptError(
+            "Encrypted payload too short to contain nonce + auth tag"
+        )
+
+    nonce = encrypted[:_XCHACHA_NONCE_LEN]
+    ciphertext_and_tag = encrypted[_XCHACHA_NONCE_LEN:]
+    ciphertext = ciphertext_and_tag[:-16]
+    tag = ciphertext_and_tag[-16:]
+
+    try:
+        cipher = ChaCha20_Poly1305.new(key=key, nonce=nonce)
+        plaintext = cipher.decrypt_and_verify(ciphertext, tag)
+    except (ValueError, KeyError) as exc:
+        raise ChunkCorruptError(
+            "Chunk authentication failed — data may be corrupt or the key is wrong"
+        ) from exc
+
+    return plaintext
+
+
+# ---------------------------------------------------------------------------
+# Chunk addressing & integrity
+# ---------------------------------------------------------------------------
+
+
+def derive_chunk_id(plaintext: bytes, hmac_key: bytes) -> str:
+    """Compute a keyed chunk address via HMAC-SHA-256.
+
+    The result is deterministic for the same ``(plaintext, hmac_key)`` pair
+    but unpredictable without knowledge of *hmac_key*, preventing offline
+    content guessing attacks.
+
+    Args:
+        plaintext: The chunk data.
+        hmac_key: 32-byte HMAC key derived from the network key.
+
+    Returns:
+        Hex-encoded HMAC-SHA-256 digest (64 hex chars).
+    """
+    return hmac.new(hmac_key, plaintext, hashlib.sha256).hexdigest()
+
+
+def compute_integrity_hash(plaintext: bytes) -> str:
+    """Compute a SHA-256 digest for post-decryption verification.
+
+    Args:
+        plaintext: The chunk data.
+
+    Returns:
+        Hex-encoded SHA-256 digest (64 hex chars).
+    """
+    return hashlib.sha256(plaintext).hexdigest()
+
+
+# ---------------------------------------------------------------------------
+# Key generation
+# ---------------------------------------------------------------------------
+
+
+def generate_network_key() -> bytes:
+    """Generate a fresh 32-byte (256-bit) random network key.
+
+    Returns:
+        Cryptographically-random 32-byte key.
+    """
+    return get_random_bytes(_KEY_LEN)
+
+
+# ---------------------------------------------------------------------------
+# Keystore (passphrase-protected key wrapping)
+# ---------------------------------------------------------------------------
+
+
+def encrypt_keystore(key: bytes, passphrase: str) -> bytes:
+    """Encrypt a network key under a passphrase using scrypt + AES-256-GCM.
+
+    Wire format::
+
+        salt (16 B) || nonce (12 B) || ciphertext || tag (16 B)
+
+    Args:
+        key: The 32-byte network key to protect.
+        passphrase: User-supplied passphrase.
+
+    Returns:
+        The encrypted keystore blob.
+    """
+    salt = get_random_bytes(_SCRYPT_SALT_LEN)
+    wrapping_key = _derive_scrypt_key(passphrase, salt)
+
+    nonce = get_random_bytes(_AES_GCM_NONCE_LEN)
+    cipher = AES.new(wrapping_key, AES.MODE_GCM, nonce=nonce)
+    ciphertext, tag = cipher.encrypt_and_digest(key)
+
+    return salt + nonce + ciphertext + tag
+
+
+def decrypt_keystore(encrypted: bytes, passphrase: str) -> bytes:
+    """Decrypt a keystore blob produced by :func:`encrypt_keystore`.
+
+    Args:
+        encrypted: The encrypted keystore blob.
+        passphrase: User-supplied passphrase.
+
+    Returns:
+        The 32-byte network key.
+
+    Raises:
+        NetworkKeyError: Wrong passphrase or corrupt keystore data.
+    """
+    min_len = _SCRYPT_SALT_LEN + _AES_GCM_NONCE_LEN + 16  # salt+nonce+tag
+    if len(encrypted) < min_len:
+        raise NetworkKeyError("Keystore data is too short or corrupt")
+
+    salt = encrypted[:_SCRYPT_SALT_LEN]
+    nonce = encrypted[_SCRYPT_SALT_LEN : _SCRYPT_SALT_LEN + _AES_GCM_NONCE_LEN]
+    ciphertext_and_tag = encrypted[_SCRYPT_SALT_LEN + _AES_GCM_NONCE_LEN :]
+    ciphertext = ciphertext_and_tag[:-16]
+    tag = ciphertext_and_tag[-16:]
+
+    wrapping_key = _derive_scrypt_key(passphrase, salt)
+
+    try:
+        cipher = AES.new(wrapping_key, AES.MODE_GCM, nonce=nonce)
+        plaintext = cipher.decrypt_and_verify(ciphertext, tag)
+    except (ValueError, KeyError) as exc:
+        raise NetworkKeyError(
+            "Failed to decrypt keystore — wrong passphrase or corrupt data"
+        ) from exc
+
+    return plaintext
+
+
+# ---------------------------------------------------------------------------
+# HKDF sub-key derivation
+# ---------------------------------------------------------------------------
+
+
+def derive_auth_token(key: bytes) -> bytes:
+    """Derive a 32-byte authentication token from *key* via HKDF-SHA256.
+
+    Args:
+        key: 32-byte network key (input keying material).
+
+    Returns:
+        32-byte derived token.
+    """
+    return _hkdf_derive(key, info=b"firecloud-auth-token")
+
+
+def derive_encryption_key(key: bytes) -> bytes:
+    """Derive a 32-byte encryption sub-key from *key* via HKDF-SHA256.
+
+    Args:
+        key: 32-byte network key (input keying material).
+
+    Returns:
+        32-byte derived key.
+    """
+    return _hkdf_derive(key, info=b"firecloud-encryption-key")
+
+
+def derive_hmac_key(key: bytes) -> bytes:
+    """Derive a 32-byte HMAC sub-key from *key* via HKDF-SHA256.
+
+    Args:
+        key: 32-byte network key (input keying material).
+
+    Returns:
+        32-byte derived key.
+    """
+    return _hkdf_derive(key, info=b"firecloud-hmac-key")
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _hkdf_derive(ikm: bytes, *, info: bytes, length: int = _KEY_LEN) -> bytes:
+    """Run HKDF-SHA256 (no salt) with the given *info* label."""
+    hkdf = HKDF(
+        algorithm=hashes.SHA256(),
+        length=length,
+        salt=None,
+        info=info,
+    )
+    return hkdf.derive(ikm)
+
+
+def _derive_scrypt_key(passphrase: str, salt: bytes) -> bytes:
+    """Derive a 32-byte wrapping key from *passphrase* using scrypt."""
+    kdf = Scrypt(
+        salt=salt,
+        length=_KEY_LEN,
+        n=_SCRYPT_N,
+        r=_SCRYPT_R,
+        p=_SCRYPT_P,
+    )
+    return kdf.derive(passphrase.encode("utf-8"))

firecloud/discovery.py

@@ -0,0 +1,164 @@
+"""FireCloud Discovery Engine.
+
+Implements mDNS peer discovery using zeroconf and config file-based peer listings
+as a fallback for LAN discovery.
+"""
+
+import asyncio
+import json
+from pathlib import Path
+import socket
+from typing import Callable
+
+from zeroconf import ServiceBrowser, ServiceInfo, ServiceListener, Zeroconf
+
+
+def get_local_ip() -> str:
+    """Determine the local IPv4 address used for network traffic."""
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    try:
+        # Try connecting to an arbitrary IP to get our routing IP address
+        s.connect(("10.254.254.254", 1))
+        ip = s.getsockname()[0]
+    except Exception:
+        ip = "127.0.0.1"
+    finally:
+        s.close()
+    return ip
+
+
+class FireCloudListener(ServiceListener):
+    """mDNS Service Listener for FireCloud peers."""
+
+    def __init__(self, discovery: "LANDiscovery") -> None:
+        self.discovery = discovery
+
+    def add_service(self, zc: Zeroconf, type_: str, name: str) -> None:
+        info = zc.get_service_info(type_, name)
+        if info:
+            self._process_info(info)
+
+    def update_service(self, zc: Zeroconf, type_: str, name: str) -> None:
+        info = zc.get_service_info(type_, name)
+        if info:
+            self._process_info(info)
+
+    def remove_service(self, zc: Zeroconf, type_: str, name: str) -> None:
+        peer_node_id = name.split(".")[0]
+        if self.discovery.on_removed_callback:
+            self.discovery.on_removed_callback(peer_node_id)
+
+    def _process_info(self, info: ServiceInfo) -> None:
+        peer_node_id = info.name.split(".")[0]
+        if peer_node_id == self.discovery.node_id:
+            return  # Skip self
+
+        # Parse properties
+        props = info.properties
+        net_id_bytes = props.get(b"network_id")
+        if not net_id_bytes:
+            return
+
+        net_id = net_id_bytes.decode("utf-8")
+        if net_id != self.discovery.network_id:
+            return  # Belong to a different network
+
+        if not info.addresses:
+            return
+
+        # Convert IP address bytes to string
+        ip = socket.inet_ntoa(info.addresses[0])
+        port = info.port
+
+        if self.discovery.on_found_callback:
+            self.discovery.on_found_callback(peer_node_id, ip, port)
+
+
+class LANDiscovery:
+    """Handles mDNS service registration and browsing on the LAN."""
+
+    def __init__(self, node_id: str, network_id: str, port: int) -> None:
+        self.node_id = node_id
+        self.network_id = network_id
+        self.port = port
+        self.zeroconf: Zeroconf | None = None
+        self.browser: ServiceBrowser | None = None
+        self.service_info: ServiceInfo | None = None
+        self.on_found_callback: Callable[[str, str, int], None] | None = None
+        self.on_removed_callback: Callable[[str], None] | None = None
+
+    async def start(self) -> None:
+        """Register the node's mDNS service and start browsing for peers."""
+        # Use run_in_executor to avoid blocking the asyncio loop during Zeroconf init
+        loop = asyncio.get_running_loop()
+        await loop.run_in_executor(None, self._sync_start)
+
+    def _sync_start(self) -> None:
+        self.zeroconf = Zeroconf()
+        local_ip = get_local_ip()
+
+        self.service_info = ServiceInfo(
+            "_firecloud._tcp.local.",
+            f"{self.node_id}._firecloud._tcp.local.",
+            addresses=[socket.inet_aton(local_ip)],
+            port=self.port,
+            properties={
+                b"version": b"0.1.0",
+                b"network_id": self.network_id.encode("utf-8"),
+            },
+        )
+        self.zeroconf.register_service(self.service_info)
+
+        listener = FireCloudListener(self)
+        self.browser = ServiceBrowser(
+            self.zeroconf, "_firecloud._tcp.local.", listener
+        )
+
+    async def stop(self) -> None:
+        """Stop browsing and unregister the service."""
+        loop = asyncio.get_running_loop()
+        await loop.run_in_executor(None, self._sync_stop)
+
+    def _sync_stop(self) -> None:
+        if self.browser:
+            self.browser.cancel()
+            self.browser = None
+        if self.zeroconf:
+            if self.service_info:
+                self.zeroconf.unregister_service(self.service_info)
+                self.service_info = None
+            self.zeroconf.close()
+            self.zeroconf = None
+
+    def on_peer_found(self, callback: Callable[[str, str, int], None]) -> None:
+        """Set the callback for when a peer is found."""
+        self.on_found_callback = callback
+
+    def on_peer_removed(self, callback: Callable[[str], None]) -> None:
+        """Set the callback for when a peer is removed."""
+        self.on_removed_callback = callback
+
+
+class PeerConfig:
+    """Manages static peer lists loaded from and saved to a config file."""
+
+    def load(self, path: Path | str) -> list[tuple[str, int]]:
+        """Load static peer endpoints from a JSON file."""
+        path = Path(path)
+        if not path.exists():
+            return []
+        try:
+            with open(path, "r") as f:
+                data = json.load(f)
+                return [(item[0], int(item[1])) for item in data]
+        except Exception:
+            return []
+
+    def save(self, path: Path | str, peers: list[tuple[str, int]]) -> None:
+        """Save a list of static peer endpoints to a JSON file."""
+        path = Path(path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        # Convert tuples to lists for standard JSON encoding
+        data = [[host, port] for host, port in peers]
+        with open(path, "w") as f:
+            json.dump(data, f, indent=4)

firecloud/distributor.py

@@ -0,0 +1,269 @@
+"""FireCloud Distributor Engine.
+
+Decides the placement of chunks (local, replicated, or erasure coded)
+based on the network peer count, and retrieves them, performing
+erasure coding reconstruction if nodes are offline.
+"""
+
+import math
+import struct
+
+from firecloud import fec
+from firecloud.crypto import derive_chunk_id, compute_integrity_hash
+from firecloud.exceptions import ChunkNotFoundError
+from firecloud.manifest import ChunkInfo
+
+# Message type constant needed for peer socket commands
+MSG_STORE_CHUNK = 0x10
+
+
+class Distributor:
+    """Orchestrates chunk distribution and retrieval strategies."""
+
+    def __init__(
+        self,
+        peers: list[str],
+        local_node_id: str,
+        fec_enabled: bool = True,
+        fec_threshold: int = 5,
+    ) -> None:
+        """Initialize the distributor.
+
+        Args:
+            peers: List of active peer node IDs.
+            local_node_id: The ID of this local node.
+            fec_enabled: Whether FEC is allowed.
+            fec_threshold: The node count threshold to use FEC.
+        """
+        self.peers = peers
+        self.local_node_id = local_node_id
+        self.fec_enabled = fec_enabled
+        self.fec_threshold = fec_threshold
+
+    def get_strategy(self) -> str:
+        """Determine the distribution strategy based on the node count."""
+        total_nodes = len(self.peers) + 1
+        if total_nodes < 2:
+            return "local"
+        elif self.fec_enabled and total_nodes >= self.fec_threshold:
+            return "erasure_coding"
+        else:
+            return "replication"
+
+    async def distribute(self, chunks: list, transport) -> list[ChunkInfo]:
+        """Distribute chunks across peers. Returns placement info.
+
+        Args:
+            chunks: List of Chunk objects containing encrypted data.
+            transport: The transport client/manager containing connections.
+        """
+        strategy = self.get_strategy()
+        all_nodes = [self.local_node_id] + self.peers
+
+        if strategy == "local":
+            chunk_infos = []
+            for c in chunks:
+                transport.node.chunk_store.store(c.chunk_id, c.data)
+                chunk_infos.append(
+                    ChunkInfo(
+                        chunk_id=c.chunk_id,
+                        integrity_hash=c.integrity_hash,
+                        index=c.index,
+                        size=len(c.data),
+                        stored_on=[self.local_node_id],
+                    )
+                )
+            return chunk_infos
+
+        elif strategy == "replication":
+            chunk_infos = []
+            for c in chunks:
+                # Store on 2 nodes (replication factor = 2) using round-robin
+                idx1 = c.index % len(all_nodes)
+                idx2 = (c.index + 1) % len(all_nodes)
+                nodes_to_store = [all_nodes[idx1], all_nodes[idx2]]
+
+                for node_id in nodes_to_store:
+                    if node_id == self.local_node_id:
+                        transport.node.chunk_store.store(c.chunk_id, c.data)
+                    else:
+                        conn = transport.node.connections.get(node_id)
+                        if conn:
+                            payload = c.chunk_id.encode("utf-8") + c.data
+                            await conn.send_message(MSG_STORE_CHUNK, payload)
+
+                chunk_infos.append(
+                    ChunkInfo(
+                        chunk_id=c.chunk_id,
+                        integrity_hash=c.integrity_hash,
+                        index=c.index,
+                        size=len(c.data),
+                        stored_on=nodes_to_store,
+                    )
+                )
+            return chunk_infos
+
+        else:
+            # erasure_coding strategy
+            k = len(chunks)
+            if k == 0:
+                return []
+            n = fec.compute_n(k)
+
+            # Prepend a header containing the chunk count and each chunk's size
+            header = struct.pack("!I", k) + b"".join(
+                struct.pack("!I", len(c.data)) for c in chunks
+            )
+            payload = header + b"".join(c.data for c in chunks)
+
+            # Encode into N shares
+            shares = fec.encode(payload, k, n)
+
+            chunk_infos = []
+            hmac_key = transport.node.network.hmac_key
+
+            for i, share_data in enumerate(shares):
+                share_id = derive_chunk_id(share_data, hmac_key)
+                share_hash = compute_integrity_hash(share_data)
+
+                # Store share on a node round-robin
+                node_id = all_nodes[i % len(all_nodes)]
+                if node_id == self.local_node_id:
+                    transport.node.chunk_store.store(share_id, share_data)
+                else:
+                    conn = transport.node.connections.get(node_id)
+                    if conn:
+                        store_payload = share_id.encode("utf-8") + share_data
+                        await conn.send_message(MSG_STORE_CHUNK, store_payload)
+
+                chunk_infos.append(
+                    ChunkInfo(
+                        chunk_id=share_id,
+                        integrity_hash=share_hash,
+                        index=i,
+                        size=len(share_data),
+                        stored_on=[node_id],
+                    )
+                )
+            return chunk_infos
+
+    async def retrieve(self, chunk_infos: list[ChunkInfo], transport) -> list[bytes]:
+        """Retrieve chunks from peers, with FEC reconstruction if needed.
+
+        Args:
+            chunk_infos: List of ChunkInfo objects representing the placement of
+                chunks or shares.
+            transport: The transport client/manager containing connections.
+        """
+        strategy = self.get_strategy()
+
+        if strategy != "erasure_coding":
+            chunks_data = []
+            for info in chunk_infos:
+                chunk_data = None
+                # Try primary stored nodes
+                for node_id in info.stored_on:
+                    if node_id == self.local_node_id:
+                        if transport.node.chunk_store.has(info.chunk_id):
+                            chunk_data = transport.node.chunk_store.retrieve(
+                                info.chunk_id
+                            )
+                            break
+                    else:
+                        conn = transport.node.connections.get(node_id)
+                        if conn:
+                            chunk_data = await conn.retrieve_chunk(info.chunk_id)
+                            if chunk_data:
+                                break
+
+                # Fallback to other connections if primary is down
+                if chunk_data is None:
+                    for node_id, conn in transport.node.connections.items():
+                        chunk_data = await conn.retrieve_chunk(info.chunk_id)
+                        if chunk_data:
+                            break
+
+                # Absolute local fallback
+                if chunk_data is None and self.local_node_id not in info.stored_on:
+                    if transport.node.chunk_store.has(info.chunk_id):
+                        chunk_data = transport.node.chunk_store.retrieve(
+                            info.chunk_id
+                        )
+
+                if chunk_data is None:
+                    raise ChunkNotFoundError(
+                        f"Failed to retrieve chunk {info.chunk_id}"
+                    )
+                chunks_data.append(chunk_data)
+            return chunks_data
+
+        else:
+            # erasure_coding strategy: chunk_infos are the N shares.
+            # Determine threshold K from N.
+            n = len(chunk_infos)
+            if n == 0:
+                return []
+            k = 1
+            while math.ceil(k * 1.5) < n:
+                k += 1
+
+            retrieved_shares = []  # list of (index, share_data)
+
+            for info in chunk_infos:
+                share_data = None
+                # Try local first
+                if self.local_node_id in info.stored_on:
+                    if transport.node.chunk_store.has(info.chunk_id):
+                        share_data = transport.node.chunk_store.retrieve(
+                            info.chunk_id
+                        )
+
+                # Try primary peer connections
+                if share_data is None:
+                    for node_id in info.stored_on:
+                        if node_id != self.local_node_id:
+                            conn = transport.node.connections.get(node_id)
+                            if conn:
+                                share_data = await conn.retrieve_chunk(
+                                    info.chunk_id
+                                )
+                                if share_data:
+                                    break
+
+                # Try generic fallback peer connections
+                if share_data is None:
+                    for node_id, conn in transport.node.connections.items():
+                        share_data = await conn.retrieve_chunk(info.chunk_id)
+                        if share_data:
+                            break
+
+                if share_data is not None:
+                    retrieved_shares.append((info.index, share_data))
+                    if len(retrieved_shares) >= k:
+                        break
+
+            if len(retrieved_shares) < k:
+                raise ChunkNotFoundError(
+                    f"Insufficient shares to reconstruct file: need {k}, got {len(retrieved_shares)}"
+                )
+
+            # Reconstruct original payload
+            payload = fec.decode(retrieved_shares, k)
+
+            # Parse header
+            num_chunks = struct.unpack("!I", payload[:4])[0]
+            sizes = []
+            offset = 4
+            for _ in range(num_chunks):
+                sizes.append(
+                    struct.unpack("!I", payload[offset : offset + 4])[0]
+                )
+                offset += 4
+
+            # Split payload into original encrypted chunks
+            chunks_data = []
+            for size in sizes:
+                chunks_data.append(payload[offset : offset + size])
+                offset += size
+
+            return chunks_data