firecloud-devnet 0.1.0__py3-none-any.whl

fc_rag/embedder.py

@@ -0,0 +1,62 @@
+"""Local text chunking and embedding via fastembed.
+
+Default model: BAAI/bge-small-en-v1.5 (384-dim, CPU-only).
+"""
+
+from functools import lru_cache
+from fastembed import TextEmbedding
+from fc_rag.config import get_settings
+
+
+@lru_cache(maxsize=1)
+def _get_model() -> TextEmbedding:
+    settings = get_settings()
+    return TextEmbedding(model_name=settings.embedding_model)
+
+
+def chunk_text(text: str, chunk_size: int = 512, overlap: int = 50) -> list[str]:
+    """Split *text* into overlapping chunks on whitespace boundaries."""
+    if not text or not text.strip():
+        return []
+
+    words = text.split()
+    chunks: list[str] = []
+    current_words: list[str] = []
+    current_len = 0
+
+    for word in words:
+        word_len = len(word)
+        addition = word_len if not current_words else word_len + 1
+
+        if current_len + addition > chunk_size and current_words:
+            chunks.append(" ".join(current_words))
+
+            # keep tail words for overlap
+            overlap_words: list[str] = []
+            overlap_len = 0
+            for w in reversed(current_words):
+                candidate = len(w) if not overlap_words else len(w) + 1
+                if overlap_len + candidate > overlap:
+                    break
+                overlap_words.insert(0, w)
+                overlap_len += candidate
+
+            current_words = overlap_words
+            current_len = overlap_len
+
+        current_words.append(word)
+        current_len += addition
+
+    if current_words:
+        chunks.append(" ".join(current_words))
+
+    return chunks
+
+
+def embed_chunks(chunks: list[str]) -> list[list[float]]:
+    """Batch-embed text chunks using the local fastembed model."""
+    if not chunks:
+        return []
+    model = _get_model()
+    embeddings = list(model.embed(chunks))
+    return [emb.tolist() for emb in embeddings]

fc_rag/indexer.py

@@ -0,0 +1,121 @@
+"""Index local files into a Qdrant vector store for RAG retrieval."""
+
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+
+from qdrant_client import QdrantClient
+from qdrant_client.models import Distance, PointStruct, VectorParams
+from rich.progress import Progress
+
+from fc_rag.config import get_settings
+from fc_rag.embedder import chunk_text, embed_chunks
+
+_SUPPORTED_EXTENSIONS = {".txt", ".md", ".py", ".json"}
+_VECTOR_DIM = 384  # BAAI/bge-small-en-v1.5
+
+
+def _ensure_collection(client: QdrantClient, name: str) -> None:
+    """Create the collection if it doesn't exist yet."""
+    existing = [c.name for c in client.get_collections().collections]
+    if name not in existing:
+        client.create_collection(
+            collection_name=name,
+            vectors_config=VectorParams(size=_VECTOR_DIM, distance=Distance.COSINE),
+        )
+
+
+def _safety_check(path: Path) -> None:
+    # block indexing of firecloud's encrypted chunk storage — those are
+    # ciphertext and would just pollute the vector store with garbage
+    resolved = path.resolve()
+    parts = resolved.parts
+
+    for i, part in enumerate(parts):
+        if part == "chunks" and i > 0 and "firecloud" in parts[i - 1].lower():
+            raise ValueError(
+                "Do not index encrypted chunk storage. "
+                "Pass your original files before encryption."
+            )
+
+    resolved_str = str(resolved).lower()
+    if "/firecloud/" in resolved_str and "chunks" in resolved_str:
+        raise ValueError(
+            "Do not index encrypted chunk storage. "
+            "Pass your original files before encryption."
+        )
+
+    if resolved.name == "chunks":
+        parent_name = resolved.parent.name.lower() if resolved.parent else ""
+        if "firecloud" in parent_name or "storage" in parent_name:
+            raise ValueError(
+                "Do not index encrypted chunk storage. "
+                "Pass your original files before encryption."
+            )
+
+
+def index_path(path: Path) -> int:
+    """Index files at *path* into the local Qdrant collection.
+
+    Only .txt, .md, .py, and .json files are processed.
+    Returns the total number of chunks indexed.
+    """
+    path = Path(path).resolve()
+    _safety_check(path)
+
+    settings = get_settings()
+    settings.qdrant_path.mkdir(parents=True, exist_ok=True)
+
+    client = QdrantClient(path=str(settings.qdrant_path))
+    _ensure_collection(client, settings.collection_name)
+
+    if path.is_file():
+        files = [path] if path.suffix in _SUPPORTED_EXTENSIONS else []
+    elif path.is_dir():
+        files = sorted(
+            f for f in path.rglob("*")
+            if f.is_file() and f.suffix in _SUPPORTED_EXTENSIONS
+        )
+    else:
+        return 0
+
+    total_chunks = 0
+
+    with Progress() as progress:
+        task = progress.add_task("[cyan]Indexing files…", total=len(files))
+
+        for filepath in files:
+            try:
+                content = filepath.read_text(encoding="utf-8", errors="replace")
+            except Exception:
+                progress.advance(task)
+                continue
+
+            chunks = chunk_text(content)
+            if not chunks:
+                progress.advance(task)
+                continue
+
+            vectors = embed_chunks(chunks)
+            now = datetime.now(timezone.utc).isoformat()
+
+            points = [
+                PointStruct(
+                    id=str(uuid.uuid4()),
+                    vector=vec,
+                    payload={
+                        "filename": filepath.name,
+                        "filepath": str(filepath),
+                        "chunk_index": i,
+                        "content": chunk,
+                        "indexed_at": now,
+                    },
+                )
+                for i, (chunk, vec) in enumerate(zip(chunks, vectors))
+            ]
+
+            client.upsert(collection_name=settings.collection_name, points=points)
+            total_chunks += len(points)
+            progress.advance(task)
+
+    return total_chunks

fc_rag/query_engine.py

@@ -0,0 +1,79 @@
+"""RAG pipeline: retrieve context → build prompt → query Ollama."""
+
+import json
+import sys
+import time
+from datetime import datetime, timezone
+
+import ollama
+from fc_rag.config import get_settings
+from fc_rag.retriever import retrieve
+
+
+def query(user_question: str) -> str:
+    """Run the full RAG pipeline and return the LLM's answer."""
+    settings = get_settings()
+    start = time.monotonic()
+
+    results = retrieve(user_question)
+
+    # build context block from retrieved chunks
+    if results:
+        ctx_parts = []
+        for i, r in enumerate(results, 1):
+            ctx_parts.append(f"[{i}] (source: {r.filename})\n{r.content}")
+        context = "\n\n".join(ctx_parts)
+    else:
+        context = "(No relevant context found.)"
+
+    messages = [
+        {
+            "role": "system",
+            "content": (
+                "Answer only from the provided context. "
+                "If the context does not contain the answer, say so."
+            ),
+        },
+        {
+            "role": "user",
+            "content": f"Context:\n{context}\n\nQuestion: {user_question}",
+        },
+    ]
+
+    answer: str | None = None
+    success = False
+
+    for attempt in range(1, settings.max_retries + 1):
+        try:
+            response = ollama.chat(
+                model=settings.ollama_model,
+                messages=messages,
+            )
+            answer = response["message"]["content"]
+            success = True
+            break
+        except (ollama.ResponseError, ConnectionError) as exc:
+            print(
+                f"[attempt {attempt}/{settings.max_retries}] Ollama error: {exc}",
+                file=sys.stderr,
+            )
+            if attempt < settings.max_retries:
+                time.sleep(2)
+
+    if not success:
+        answer = "Local LLM unavailable. Start Ollama with: ollama serve"
+
+    elapsed_ms = (time.monotonic() - start) * 1000
+
+    # append to query log
+    settings.log_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(settings.log_path, "a", encoding="utf-8") as fh:
+        fh.write(json.dumps({
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "question_length": len(user_question),
+            "chunks_retrieved": len(results),
+            "latency_ms": round(elapsed_ms, 2),
+            "success": success,
+        }, default=str) + "\n")
+
+    return answer

fc_rag/requirements.txt

@@ -0,0 +1,6 @@
+qdrant-client
+fastembed
+ollama
+pydantic>=2.0
+click
+rich

fc_rag/retriever.py

@@ -0,0 +1,46 @@
+"""Search the local Qdrant collection for relevant chunks."""
+
+from pydantic import BaseModel, ConfigDict
+from qdrant_client import QdrantClient
+
+from fc_rag.config import get_settings
+from fc_rag.embedder import embed_chunks
+
+
+class RetrievalResult(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    content: str
+    filename: str
+    score: float
+    chunk_index: int
+
+
+def retrieve(query: str, top_k: int | None = None) -> list[RetrievalResult]:
+    """Embed *query* and return the closest chunks from Qdrant."""
+    settings = get_settings()
+    k = top_k if top_k is not None else settings.top_k
+
+    vectors = embed_chunks([query])
+    if not vectors:
+        return []
+    query_vector = vectors[0]
+
+    client = QdrantClient(path=str(settings.qdrant_path))
+
+    response = client.query_points(
+        collection_name=settings.collection_name,
+        query=query_vector,
+        limit=k,
+    )
+    results = response.points
+
+    return [
+        RetrievalResult(
+            content=hit.payload.get("content", ""),
+            filename=hit.payload.get("filename", "unknown"),
+            score=hit.score,
+            chunk_index=hit.payload.get("chunk_index", 0),
+        )
+        for hit in results
+    ]

firecloud/__init__.py

@@ -0,0 +1,17 @@
+"""
+FireCloud — Private, encrypted, distributed storage across your own machines.
+
+Usage:
+    from firecloud import Node, Network
+
+    net = Network.create(passphrase="your-passphrase")
+    node = Node(network=net, storage_path="~/.firecloud/storage")
+    node.start()
+"""
+
+__version__ = "0.1.0"
+
+from firecloud.network import Network
+from firecloud.node import Node
+
+__all__ = ["Node", "Network", "__version__"]

firecloud/chunker.py

@@ -0,0 +1,122 @@
+"""FireCloud Chunker Engine.
+
+Wraps FastCDC for content-defined chunking with keyed chunk addressing
+and integrity verification.
+"""
+
+from dataclasses import dataclass
+import hashlib
+import hmac
+from pathlib import Path
+from fastcdc import fastcdc
+
+from firecloud.crypto import derive_chunk_id, compute_integrity_hash
+
+
+@dataclass
+class Chunk:
+    """Represents a content-defined chunk of a file."""
+    index: int
+    offset: int
+    length: int
+    data: bytes
+    chunk_id: str        # HMAC-SHA-256 (keyed address)
+    integrity_hash: str  # SHA-256 (for verification)
+
+
+def chunk_file(
+    filepath: Path | str,
+    hmac_key: bytes,
+    min_size: int = 4096,
+    avg_size: int = 16384,
+    max_size: int = 65536,
+) -> list[Chunk]:
+    """Read a file and chunk it with FastCDC.
+
+    Computes the keyed chunk_id and integrity_hash for each chunk.
+    """
+    path = Path(filepath)
+    if not path.exists() or path.stat().st_size == 0:
+        return []
+        
+    # fastcdc expects string filepath or bytes.
+    # We use fat=True so that c.data contains the actual chunk bytes.
+    cdc_chunks = fastcdc(
+        str(path),
+        min_size=min_size,
+        avg_size=avg_size,
+        max_size=max_size,
+        fat=True,
+    )
+    
+    chunks = []
+    for index, c in enumerate(cdc_chunks):
+        chunk_data = c.data
+        chunk_id = derive_chunk_id(chunk_data, hmac_key)
+        int_hash = compute_integrity_hash(chunk_data)
+        chunks.append(
+            Chunk(
+                index=index,
+                offset=c.offset,
+                length=c.length,
+                data=chunk_data,
+                chunk_id=chunk_id,
+                integrity_hash=int_hash,
+            )
+        )
+    return chunks
+
+
+def chunk_bytes(
+    data: bytes,
+    hmac_key: bytes,
+    min_size: int = 4096,
+    avg_size: int = 16384,
+    max_size: int = 65536,
+) -> list[Chunk]:
+    """Chunk in-memory bytes using FastCDC."""
+    if not data:
+        return []
+        
+    cdc_chunks = fastcdc(
+        data,
+        min_size=min_size,
+        avg_size=avg_size,
+        max_size=max_size,
+        fat=True,
+    )
+    
+    chunks = []
+    for index, c in enumerate(cdc_chunks):
+        chunk_data = c.data
+        chunk_id = derive_chunk_id(chunk_data, hmac_key)
+        int_hash = compute_integrity_hash(chunk_data)
+        chunks.append(
+            Chunk(
+                index=index,
+                offset=c.offset,
+                length=c.length,
+                data=chunk_data,
+                chunk_id=chunk_id,
+                integrity_hash=int_hash,
+            )
+        )
+    return chunks
+
+
+def reassemble_chunks(chunks: list[Chunk]) -> bytes:
+    """Reassemble chunks in index order back to original bytes."""
+    sorted_chunks = sorted(chunks, key=lambda c: c.index)
+    return b"".join(c.data for c in sorted_chunks)
+
+
+def compute_file_id(filepath: Path | str, hmac_key: bytes) -> str:
+    """Compute the HMAC-SHA-256 of the entire file content."""
+    h = hmac.new(hmac_key, digestmod=hashlib.sha256)
+    with open(filepath, "rb") as f:
+        while True:
+            block = f.read(65536)
+            if not block:
+                break
+            h.update(block)
+    return h.hexdigest()