firecloud-devnet 0.1.0__tar.gz → 0.2.0__tar.gz

{firecloud_devnet-0.1.0 → firecloud_devnet-0.2.0}/.gitignore

@@ -38,8 +38,9 @@ htmlcov/
 .DS_Store
 Thumbs.db
 
-# AI tool metadata — do not commit
-.antigravitycli/
-
 # Qdrant local data (created by fc-rag)
 ~/.fc_rag/
+
+# Test configurations
+test_config/
+

firecloud_devnet-0.2.0/CHANGELOG.md

@@ -0,0 +1,64 @@
+# Changelog
+
+All notable changes to FireCloud will be documented in this file.
+
+Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
+
+## [Unreleased]
+
+### Fixed
+- **Erasure-coded files are now downloadable after the cluster shrinks.**
+  Download derived the placement strategy from the *live* peer count, so a
+  file uploaded with erasure coding became unreadable once fewer than 5
+  nodes were online — even with enough shares reachable. The strategy and
+  reconstruction threshold now come from the manifest entry.
+- Manifest merges with equal Lamport timestamps now converge on a
+  deterministic winner (tombstone first, then uploader node ID) instead of
+  each node keeping its own version forever.
+- Chunk stores on peers are now acknowledged; placement metadata
+  (`stored_on`) only records nodes that actually persisted the chunk, and
+  upload falls back to other nodes when a peer is full.
+- Chunk retrieval responses are matched to requests by chunk ID, so a late
+  reply can no longer be delivered to the wrong request; retrieval and
+  handshake have timeouts instead of hanging forever on a silent peer.
+- Frame length on the wire is capped (64 MiB) so a corrupt or hostile
+  length prefix cannot trigger a multi-gigabyte allocation.
+- Corrupt FEC shares are detected via their integrity hash and skipped
+  during reconstruction instead of poisoning the decoded file.
+- Chunk writes are atomic (write-temp-then-rename) and storage usage is
+  tracked incrementally instead of re-walking the chunk tree on every store.
+- Manifest and artifact-store JSON files are written atomically.
+- Auth token comparison is constant-time.
+- Re-saving an `fc-ml` artifact with the same name+version replaces the
+  record instead of duplicating it.
+- `fc-rag` re-indexing no longer duplicates vector points (deterministic
+  point IDs + per-file cleanup), and indexing/retrieval share one embedded
+  Qdrant client.
+
+### Added
+- `firecloud verify [FILE_ID]` — checks chunk availability and integrity
+  across the network and reports each file as healthy / degraded /
+  unrecoverable (non-zero exit on unrecoverable files).
+- `FIRECLOUD_PASSPHRASE`, `FIRECLOUD_DATA_DIR`, and
+  `FIRECLOUD_MAX_STORAGE_GB` environment variables are honored by the CLI
+  (docker-compose already set them; the code now reads them).
+- `--bootstrap host:port` option on `start`, `upload`, `download`,
+  `delete`, `verify`, and `sync` to connect to a peer without mDNS.
+- Peer-side store/has-chunk protocol messages (`STORE_OK`, `STORE_FAIL`,
+  `HAS_CHUNK`) backing the fixes above.
+
+## [0.1.0] - 2025-06-01
+
+### Added
+- XChaCha20-Poly1305 chunk encryption with HMAC-SHA-256 keyed addressing
+- FastCDC content-defined chunking
+- zfec erasure coding (configurable k/n)
+- mDNS peer discovery via zeroconf with config file fallback
+- TLS binary RPC transport with handshake and heartbeat
+- Manifest with Lamport timestamps and tombstone support
+- Watchdog-based folder sync (outbound upload, inbound download)
+- Click CLI (`firecloud` entry point) — init, start, upload, download, status, peers
+- Docker Compose multi-node setup with health checks
+- GitHub Actions CI (lint → test → build)
+- `fc-rag`: local RAG pipeline (fastembed + Qdrant + Ollama)
+- `fc-ml`: ML artifact versioning, telemetry server, anomaly detection

firecloud_devnet-0.2.0/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+RUN pip install --no-cache-dir --upgrade pip
+
+COPY . .
+RUN pip install --no-cache-dir .
+
+EXPOSE 7474
+
+ENTRYPOINT ["firecloud"]

{firecloud_devnet-0.1.0 → firecloud_devnet-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: firecloud-devnet
-Version: 0.1.0
+Version: 0.2.0
 Summary: Private, encrypted, distributed storage across your own machines
 Project-URL: Homepage, https://github.com/rajashekharsunkara/firecloud
 Project-URL: Repository, https://github.com/rajashekharsunkara/firecloud
@@ -87,8 +87,27 @@ docker exec firecloud-bootstrap firecloud upload /data/my-file.zip
 
 # 3. Download from any node
 docker exec firecloud-node-1 firecloud download <file_id> /data/restored.zip
+
+# 4. Check replica/share health across the network
+docker exec firecloud-bootstrap firecloud verify
+```
+
+### CLI essentials
+
+```bash
+firecloud init                          # create a network keystore
+firecloud start --bootstrap host:7474   # run a node, optionally joining a peer
+firecloud upload <path>                 # chunk → encrypt → distribute
+firecloud download <file_id> <output>   # retrieve → verify → reassemble
+firecloud verify [file_id]              # health report: healthy / degraded / unrecoverable
+firecloud sync <folder>                 # bi-directional folder sync
 ```
 
+Environment variables (all optional): `FIRECLOUD_PASSPHRASE` (skip the
+passphrase prompt), `FIRECLOUD_DATA_DIR` (default storage directory),
+`FIRECLOUD_MAX_STORAGE_GB` (chunk-store quota), and `FIRECLOUD_BOOTSTRAP`
+(comma-separated peers to connect to on start).
+
 ---
 
 ## Architecture
@@ -116,6 +135,10 @@ docker exec firecloud-node-1 firecloud download <file_id> /data/restored.zip
 
 FireCloud uses **HMAC-SHA-256 with a network-derived key** for chunk addressing instead of plain SHA-256. This raises the cost of confirmation-of-file attacks — an attacker who suspects a specific file is stored cannot verify its presence by computing chunk hashes from the plaintext, because valid chunk IDs require the network key. This protection holds as long as the network key remains confidential.
 
+Chunks are encrypted with XChaCha20-Poly1305 before leaving the machine, so storage nodes only ever see authenticated ciphertext. Transport hardening includes a frame-size cap, handshake and request timeouts, and constant-time auth-token comparison.
+
+**Known limitations (devnet):** node TLS certificates are self-signed and clients do not verify them, so the TLS layer protects against passive snooping but not an active man-in-the-middle on your LAN; the chunk payloads themselves remain end-to-end encrypted regardless. Deploy only on networks you control.
+
 ---
 
 ## AI/ML Extensions

{firecloud_devnet-0.1.0 → firecloud_devnet-0.2.0}/README.md

@@ -35,8 +35,27 @@ docker exec firecloud-bootstrap firecloud upload /data/my-file.zip
 
 # 3. Download from any node
 docker exec firecloud-node-1 firecloud download <file_id> /data/restored.zip
+
+# 4. Check replica/share health across the network
+docker exec firecloud-bootstrap firecloud verify
+```
+
+### CLI essentials
+
+```bash
+firecloud init                          # create a network keystore
+firecloud start --bootstrap host:7474   # run a node, optionally joining a peer
+firecloud upload <path>                 # chunk → encrypt → distribute
+firecloud download <file_id> <output>   # retrieve → verify → reassemble
+firecloud verify [file_id]              # health report: healthy / degraded / unrecoverable
+firecloud sync <folder>                 # bi-directional folder sync
 ```
 
+Environment variables (all optional): `FIRECLOUD_PASSPHRASE` (skip the
+passphrase prompt), `FIRECLOUD_DATA_DIR` (default storage directory),
+`FIRECLOUD_MAX_STORAGE_GB` (chunk-store quota), and `FIRECLOUD_BOOTSTRAP`
+(comma-separated peers to connect to on start).
+
 ---
 
 ## Architecture
@@ -64,6 +83,10 @@ docker exec firecloud-node-1 firecloud download <file_id> /data/restored.zip
 
 FireCloud uses **HMAC-SHA-256 with a network-derived key** for chunk addressing instead of plain SHA-256. This raises the cost of confirmation-of-file attacks — an attacker who suspects a specific file is stored cannot verify its presence by computing chunk hashes from the plaintext, because valid chunk IDs require the network key. This protection holds as long as the network key remains confidential.
 
+Chunks are encrypted with XChaCha20-Poly1305 before leaving the machine, so storage nodes only ever see authenticated ciphertext. Transport hardening includes a frame-size cap, handshake and request timeouts, and constant-time auth-token comparison.
+
+**Known limitations (devnet):** node TLS certificates are self-signed and clients do not verify them, so the TLS layer protects against passive snooping but not an active man-in-the-middle on your LAN; the chunk payloads themselves remain end-to-end encrypted regardless. Deploy only on networks you control.
+
 ---
 
 ## AI/ML Extensions

{firecloud_devnet-0.1.0 → firecloud_devnet-0.2.0}/docker-compose.yml

@@ -6,6 +6,7 @@ services:
       - "7474:7474"
     volumes:
       - fc-bootstrap-data:/data
+      - ./test_config/.firecloud:/root/.firecloud:ro,z
     environment:
       - FIRECLOUD_PASSPHRASE=${FIRECLOUD_PASSPHRASE}
       - FIRECLOUD_MAX_STORAGE_GB=${FIRECLOUD_MAX_STORAGE_GB:-10}
@@ -27,6 +28,7 @@ services:
       - "7475:7475"
     volumes:
       - fc-node1-data:/data
+      - ./test_config/.firecloud:/root/.firecloud:ro,z
     environment:
       - FIRECLOUD_PASSPHRASE=${FIRECLOUD_PASSPHRASE}
       - FIRECLOUD_BOOTSTRAP=bootstrap-node:7474
@@ -51,6 +53,7 @@ services:
       - "7476:7476"
     volumes:
       - fc-node2-data:/data
+      - ./test_config/.firecloud:/root/.firecloud:ro,z
     environment:
       - FIRECLOUD_PASSPHRASE=${FIRECLOUD_PASSPHRASE}
       - FIRECLOUD_BOOTSTRAP=bootstrap-node:7474
@@ -75,6 +78,7 @@ services:
       - "7477:7477"
     volumes:
       - fc-node3-data:/data
+      - ./test_config/.firecloud:/root/.firecloud:ro,z
     environment:
       - FIRECLOUD_PASSPHRASE=${FIRECLOUD_PASSPHRASE}
       - FIRECLOUD_BOOTSTRAP=bootstrap-node:7474

{firecloud_devnet-0.1.0 → firecloud_devnet-0.2.0}/fc_mlops/artifact_store.py

@@ -36,10 +36,13 @@ def _load_manifest() -> list[dict]:
 
 def _save_manifest(entries: list[dict]) -> None:
     _MANIFEST_PATH.parent.mkdir(parents=True, exist_ok=True)
-    _MANIFEST_PATH.write_text(
+    # Write-then-rename so a crash mid-write cannot truncate the manifest.
+    tmp_path = _MANIFEST_PATH.with_name(_MANIFEST_PATH.name + ".tmp")
+    tmp_path.write_text(
         json.dumps(entries, indent=2, default=str),
         encoding="utf-8",
     )
+    tmp_path.replace(_MANIFEST_PATH)
 
 
 async def save_artifact(
@@ -70,6 +73,12 @@ async def save_artifact(
     )
 
     entries = _load_manifest()
+    # Re-saving the same name+version replaces the old record — otherwise
+    # load_artifact would keep returning the stale first match forever.
+    entries = [
+        e for e in entries
+        if not (e.get("name") == name and e.get("version") == version)
+    ]
     entries.append(metadata.model_dump())
     _save_manifest(entries)
     return metadata

{firecloud_devnet-0.1.0 → firecloud_devnet-0.2.0}/fc_mlops/simulate_failure.py

@@ -62,7 +62,7 @@ def main() -> None:
 
     # inject failures
     console.print("[bold yellow][Phase 2][/bold yellow] Injecting failure signatures...")
-    for _ in range(10):
+    for _ in range(3):
         _write_reading(log_path, _anomalous_reading())
 
     # detect

{firecloud_devnet-0.1.0 → firecloud_devnet-0.2.0}/fc_rag/cli.py

@@ -36,14 +36,12 @@ def index(path: str):
 def query(question: str):
     """Query the local RAG pipeline with a natural-language question."""
     from fc_rag.query_engine import query as run_query
-    from fc_rag.retriever import retrieve
 
-    answer = run_query(question)
-    click.echo(answer)
+    response = run_query(question)
+    click.echo(response.answer)
 
-    results = retrieve(question)
-    if results:
-        sources = sorted(set(r.filename for r in results))
+    if response.sources:
+        sources = sorted(set(r.filename for r in response.sources))
         click.echo(f"\nSources: {', '.join(sources)}")
 
 

{firecloud_devnet-0.1.0 → firecloud_devnet-0.2.0}/fc_rag/config.py

@@ -22,3 +22,18 @@ class Settings(BaseModel):
 @lru_cache(maxsize=1)
 def get_settings() -> Settings:
     return Settings()
+
+
+@lru_cache(maxsize=1)
+def get_client():
+    """Shared embedded-Qdrant client.
+
+    Embedded Qdrant locks its storage directory per client instance, so
+    indexing and retrieval within one process must reuse a single client
+    rather than each opening their own.
+    """
+    from qdrant_client import QdrantClient
+
+    settings = get_settings()
+    settings.qdrant_path.mkdir(parents=True, exist_ok=True)
+    return QdrantClient(path=str(settings.qdrant_path))

{firecloud_devnet-0.1.0 → firecloud_devnet-0.2.0}/fc_rag/indexer.py

@@ -5,10 +5,18 @@ from datetime import datetime, timezone
 from pathlib import Path
 
 from qdrant_client import QdrantClient
-from qdrant_client.models import Distance, PointStruct, VectorParams
+from qdrant_client.models import (
+    Distance,
+    FieldCondition,
+    Filter,
+    FilterSelector,
+    MatchValue,
+    PointStruct,
+    VectorParams,
+)
 from rich.progress import Progress
 
-from fc_rag.config import get_settings
+from fc_rag.config import get_client, get_settings
 from fc_rag.embedder import chunk_text, embed_chunks
 
 _SUPPORTED_EXTENSIONS = {".txt", ".md", ".py", ".json"}
@@ -64,9 +72,7 @@ def index_path(path: Path) -> int:
     _safety_check(path)
 
     settings = get_settings()
-    settings.qdrant_path.mkdir(parents=True, exist_ok=True)
-
-    client = QdrantClient(path=str(settings.qdrant_path))
+    client = get_client()
     _ensure_collection(client, settings.collection_name)
 
     if path.is_file():
@@ -99,9 +105,28 @@ def index_path(path: Path) -> int:
             vectors = embed_chunks(chunks)
             now = datetime.now(timezone.utc).isoformat()
 
+            # Drop previously indexed points for this file so re-indexing
+            # cannot accumulate duplicates or leave stale trailing chunks.
+            client.delete(
+                collection_name=settings.collection_name,
+                points_selector=FilterSelector(
+                    filter=Filter(
+                        must=[
+                            FieldCondition(
+                                key="filepath",
+                                match=MatchValue(value=str(filepath)),
+                            )
+                        ]
+                    )
+                ),
+            )
+
             points = [
                 PointStruct(
-                    id=str(uuid.uuid4()),
+                    # Deterministic ID per (file, chunk position) — an
+                    # upsert of the same file overwrites instead of
+                    # duplicating.
+                    id=str(uuid.uuid5(uuid.NAMESPACE_URL, f"{filepath}::{i}")),
                     vector=vec,
                     payload={
                         "filename": filepath.name,

{firecloud_devnet-0.1.0 → firecloud_devnet-0.2.0}/fc_rag/query_engine.py

@@ -6,12 +6,23 @@ import time
 from datetime import datetime, timezone
 
 import ollama
+from pydantic import BaseModel, ConfigDict
+
 from fc_rag.config import get_settings
-from fc_rag.retriever import retrieve
+from fc_rag.retriever import RetrievalResult, retrieve
+
+
+class QueryResponse(BaseModel):
+    """Answer plus the retrieved chunks it was grounded on."""
+
+    model_config = ConfigDict(frozen=True)
+
+    answer: str
+    sources: list[RetrievalResult]
 
 
-def query(user_question: str) -> str:
-    """Run the full RAG pipeline and return the LLM's answer."""
+def query(user_question: str) -> QueryResponse:
+    """Run the full RAG pipeline and return the answer with its sources."""
     settings = get_settings()
     start = time.monotonic()
 
@@ -76,4 +87,4 @@ def query(user_question: str) -> str:
             "success": success,
         }, default=str) + "\n")
 
-    return answer
+    return QueryResponse(answer=answer, sources=results)

{firecloud_devnet-0.1.0 → firecloud_devnet-0.2.0}/fc_rag/retriever.py

@@ -1,9 +1,8 @@
 """Search the local Qdrant collection for relevant chunks."""
 
 from pydantic import BaseModel, ConfigDict
-from qdrant_client import QdrantClient
 
-from fc_rag.config import get_settings
+from fc_rag.config import get_client, get_settings
 from fc_rag.embedder import embed_chunks
 
 
@@ -26,13 +25,17 @@ def retrieve(query: str, top_k: int | None = None) -> list[RetrievalResult]:
         return []
     query_vector = vectors[0]
 
-    client = QdrantClient(path=str(settings.qdrant_path))
+    client = get_client()
 
-    response = client.query_points(
-        collection_name=settings.collection_name,
-        query=query_vector,
-        limit=k,
-    )
+    try:
+        response = client.query_points(
+            collection_name=settings.collection_name,
+            query=query_vector,
+            limit=k,
+        )
+    except Exception:
+        # Collection does not exist yet — nothing has been indexed.
+        return []
     results = response.points
 
     return [