ragx-cli 0.1.0__py3-none-any.whl

ragx/core/config.py

@@ -0,0 +1,130 @@
+"""Config: .ragx/config.toml is the single source of truth. Written by `init`, read everywhere."""
+
+from __future__ import annotations
+
+import tomllib
+from pathlib import Path
+from typing import Any
+
+import tomli_w
+
+from ragx.core.errors import NotInitializedError, RagxError
+
+RAGX_DIR = ".ragx"
+
+DEFAULTS: dict[str, dict[str, Any]] = {
+    "corpus": {"include": ["**/*"], "exclude": [], "respect_gitignore": True},
+    "chunking": {"size_tokens": 800, "overlap": 0.15},
+    "graph": {"k": 8, "min_edge_sim": 0.55},
+    "traversal": {"hops": 2, "decay": 0.5, "query_floor": 0.35, "max_frontier": 150},
+    "fusion": {"rrf_k": 60, "per_query_top": 20},
+    "scoring": {"alpha_rerank": 0.6, "beta_heat": 0.25, "gamma_vector": 0.15},
+    "embeddings": {
+        "provider": "openai",
+        "base_url": "http://localhost:1234/v1",
+        "model": "text-embedding-nomic-embed-text-v1.5@q4_k_m",
+        "doc_prefix": "search_document: ",
+        "query_prefix": "search_query: ",
+        "batch_size": 32,
+        "api_key_env": "",  # NAME of an env var holding the API key; empty = no auth header
+    },
+    "expansion": {
+        "enabled": True,
+        "provider": "openai",
+        "base_url": "http://localhost:1234/v1",
+        "model": "mlx-community/Ornith-1.0-35B-3bit",
+        "variants": 3,
+        "hyde": True,
+        "api_key_env": "",
+    },
+    "rerank": {"enabled": True, "provider": "sentence-transformers", "model": "BAAI/bge-reranker-v2-m3"},
+    "query": {"top": 8, "max_chunk_chars": 1200},
+}
+
+
+def find_root(start: Path | None = None) -> Path | None:
+    """Walk up from `start` (default cwd) looking for a .ragx/ directory."""
+    cur = (start or Path.cwd()).resolve()
+    for p in [cur, *cur.parents]:
+        if (p / RAGX_DIR).is_dir():
+            return p
+    return None
+
+
+def require_root(start: Path | None = None) -> Path:
+    root = find_root(start)
+    if root is None:
+        raise NotInitializedError("no .ragx/ found — run `ragx init` first")
+    return root
+
+
+def config_path(root: Path) -> Path:
+    return root / RAGX_DIR / "config.toml"
+
+
+def db_path(root: Path) -> Path:
+    return root / RAGX_DIR / "index.db"
+
+
+def vectors_path(root: Path) -> Path:
+    return root / RAGX_DIR / "vectors.hnsw"
+
+
+def _deep_merge(base: dict, override: dict) -> dict:
+    out = {k: dict(v) for k, v in base.items()}
+    for section, values in override.items():
+        out.setdefault(section, {}).update(values)
+    return out
+
+
+class Config:
+    """Two-level (section.key) config with defaults merged under the file's values."""
+
+    def __init__(self, data: dict[str, dict[str, Any]]):
+        self.data = data
+
+    @classmethod
+    def load(cls, root: Path) -> Config:
+        path = config_path(root)
+        file_data: dict = {}
+        if path.exists():
+            with path.open("rb") as f:
+                file_data = tomllib.load(f)
+        return cls(_deep_merge(DEFAULTS, file_data))
+
+    def get(self, dotted: str) -> Any:
+        section, _, key = dotted.partition(".")
+        if not key:
+            raise RagxError(f"config keys are 'section.key', got: {dotted!r}")
+        try:
+            return self.data[section][key]
+        except KeyError:
+            raise RagxError(f"unknown config key: {dotted!r}") from None
+
+    def set(self, dotted: str, value: Any) -> None:
+        section, _, key = dotted.partition(".")
+        if not key or section not in DEFAULTS or key not in DEFAULTS[section]:
+            raise RagxError(f"unknown config key: {dotted!r}")
+        current = DEFAULTS[section][key]
+        if isinstance(current, bool):
+            value = str(value).lower() in ("1", "true", "yes", "on")
+        elif isinstance(current, int) and not isinstance(current, bool):
+            value = int(value)
+        elif isinstance(current, float):
+            value = float(value)
+        elif isinstance(current, list) and isinstance(value, str):
+            value = [v.strip() for v in value.split(",") if v.strip()]
+        self.data.setdefault(section, {})[key] = value
+
+    def save(self, root: Path) -> None:
+        path = config_path(root)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with path.open("wb") as f:
+            tomli_w.dump(self.data, f)
+
+
+def write_default_config(root: Path) -> Path:
+    """Create .ragx/ with a default config.toml. Used by `ragx init`."""
+    cfg = Config({k: dict(v) for k, v in DEFAULTS.items()})
+    cfg.save(root)
+    return config_path(root)

ragx/core/discovery.py

@@ -0,0 +1,79 @@
+"""File discovery: walk a corpus root, filter by globs/.gitignore, skip binaries."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pathspec
+import xxhash
+
+MAX_FILE_SIZE = 2 * 1024 * 1024
+BINARY_SNIFF_BYTES = 8192
+ALWAYS_SKIP_DIRS = {".ragx", ".git", "node_modules", "__pycache__", ".venv", "venv"}
+
+
+def _is_binary(path: Path) -> bool:
+    with path.open("rb") as f:
+        chunk = f.read(BINARY_SNIFF_BYTES)
+    return b"\x00" in chunk
+
+
+def _load_gitignore_spec(root: Path) -> pathspec.PathSpec | None:
+    gitignore = root / ".gitignore"
+    if not gitignore.is_file():
+        return None
+    lines = gitignore.read_text(encoding="utf-8", errors="ignore").splitlines()
+    return pathspec.PathSpec.from_lines("gitwildmatch", lines)
+
+
+def discover_files(
+    root: Path,
+    include: list[str],
+    exclude: list[str],
+    respect_gitignore: bool = True,
+) -> list[str]:
+    """Return sorted relative POSIX paths of files matching include/exclude under root.
+
+    Always skips ALWAYS_SKIP_DIRS, hidden directories, binaries (NUL byte in first 8KB),
+    and files larger than 2MB. Only a root-level .gitignore is honored (MVP limitation).
+    """
+    include_spec = pathspec.PathSpec.from_lines("gitwildmatch", include)
+    exclude_spec = pathspec.PathSpec.from_lines("gitwildmatch", exclude) if exclude else None
+    gitignore_spec = _load_gitignore_spec(root) if respect_gitignore else None
+
+    results: list[str] = []
+    for dirpath, dirnames, filenames in root.walk():
+        dirnames[:] = [
+            d for d in dirnames if d not in ALWAYS_SKIP_DIRS and not d.startswith(".")
+        ]
+        for name in filenames:
+            path = dirpath / name
+            rel = path.relative_to(root).as_posix()
+
+            if not include_spec.match_file(rel):
+                continue
+            if exclude_spec is not None and exclude_spec.match_file(rel):
+                continue
+            if gitignore_spec is not None and gitignore_spec.match_file(rel):
+                continue
+
+            try:
+                if path.stat().st_size > MAX_FILE_SIZE:
+                    continue
+                if _is_binary(path):
+                    continue
+            except OSError:
+                continue
+
+            results.append(rel)
+
+    return sorted(results)
+
+
+def hash_file(path: Path) -> str:
+    """xxhash.xxh64 hexdigest of the file's raw bytes."""
+    hasher = xxhash.xxh64()
+    with path.open("rb") as f:
+        for block in iter(lambda: f.read(65536), b""):
+            hasher.update(block)
+    return hasher.hexdigest()

ragx/core/errors.py

@@ -0,0 +1,10 @@
+class RagxError(Exception):
+    """Base error for ragx. CLI maps this to exit code 2."""
+
+
+class NotInitializedError(RagxError):
+    """No .ragx/ directory found at or above the working directory."""
+
+
+class ManifestMismatchError(RagxError):
+    """Index was built with a different embedding model/dimension than configured."""

ragx/core/eval.py

@@ -0,0 +1,93 @@
+"""Evaluation harness: compare retrieval configs against a labeled queries.jsonl file.
+
+See CONTRACTS-PHASE23.md Module I. Decoupled from the pipeline via an injected QueryFn.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable
+from pathlib import Path
+
+from ragx.core.errors import RagxError
+from ragx.core.query import QueryOptions, QueryOutput, to_files_json
+
+QueryFn = Callable[[str, QueryOptions], QueryOutput]
+
+
+def load_queries(path: Path) -> list[dict]:
+    """Parse `{"query": str, "relevant_files": [str, ...]}` per non-blank line."""
+    if not path.exists():
+        raise RagxError(f"queries file not found: {path}")
+    queries: list[dict] = []
+    for lineno, raw in enumerate(path.read_text().splitlines(), start=1):
+        line = raw.strip()
+        if not line:
+            continue
+        try:
+            obj = json.loads(line)
+        except json.JSONDecodeError as exc:
+            raise RagxError(f"malformed query on line {lineno}: {exc}") from exc
+        if not isinstance(obj, dict) or "query" not in obj or "relevant_files" not in obj:
+            raise RagxError(f"malformed query on line {lineno}: expected 'query' and 'relevant_files'")
+        queries.append(obj)
+    return queries
+
+
+def _recall(ranked: list[str], relevant: set[str], k: int) -> float:
+    if not relevant:
+        return 0.0
+    hits = len(set(ranked[:k]) & relevant)
+    return hits / len(relevant)
+
+
+def _hit_ranks(ranked: list[str], relevant: set[str]) -> list[int]:
+    return [i + 1 for i, f in enumerate(ranked) if f in relevant]
+
+
+def evaluate(
+    queries: list[dict],
+    configs: list[tuple[str, QueryOptions]],
+    query_fn: QueryFn,
+    *,
+    top: int = 10,
+) -> dict:
+    """Run each config against each query; rank files via `to_files_json`, average metrics.
+
+    `top` bounds the ranked-files window that recall@5/@10/mrr are computed over.
+    """
+    totals = {name: {"recall_at_5": 0.0, "recall_at_10": 0.0, "mrr": 0.0} for name, _ in configs}
+    per_query: list[dict] = []
+    n = len(queries)
+
+    for q in queries:
+        relevant = set(q["relevant_files"])
+        per_config: dict[str, dict] = {}
+        for name, opts in configs:
+            out = query_fn(q["query"], opts)
+            ranked = [f["file"] for f in to_files_json(out)["files"]][:top]
+            hit_ranks = _hit_ranks(ranked, relevant)
+            mrr = 1.0 / hit_ranks[0] if hit_ranks else 0.0
+            totals[name]["recall_at_5"] += _recall(ranked, relevant, 5)
+            totals[name]["recall_at_10"] += _recall(ranked, relevant, 10)
+            totals[name]["mrr"] += mrr
+            per_config[name] = {"hit_ranks": hit_ranks}
+        per_query.append({"query": q["query"], "per_config": per_config})
+
+    configs_out = [
+        {
+            "name": name,
+            "recall_at_5": totals[name]["recall_at_5"] / n if n else 0.0,
+            "recall_at_10": totals[name]["recall_at_10"] / n if n else 0.0,
+            "mrr": totals[name]["mrr"] / n if n else 0.0,
+        }
+        for name, _ in configs
+    ]
+
+    return {
+        "schema": "ragx.eval.v1",
+        "top": top,
+        "query_count": n,
+        "configs": configs_out,
+        "queries": per_query,
+    }

ragx/core/expansion.py

@@ -0,0 +1,89 @@
+"""Query expansion: LLM-generated reformulations + a HyDE passage, one generate() call."""
+
+from __future__ import annotations
+
+import json
+import sys
+from dataclasses import dataclass
+
+from ragx.providers.base import Generator
+
+_SYSTEM = (
+    "You rewrite search queries for a code/document retrieval system. "
+    "Respond with strict JSON only, no markdown, no commentary."
+)
+
+
+@dataclass
+class Expansion:
+    variants: list[str]  # reformulations/sub-queries, excludes the original
+    hyde: str | None
+
+
+def _build_prompt(query: str, variants: int, hyde: bool) -> str:
+    lines = [
+        f'Given the search query: "{query}"',
+        f"Produce up to {variants} alternative phrasings or related sub-queries "
+        "that would help retrieve relevant passages.",
+    ]
+    if hyde:
+        lines.append(
+            "Also produce a short hypothetical passage that would answer the query "
+            "(HyDE-style)."
+        )
+        schema = '{"queries": ["...", "..."], "hyde": "..."}'
+    else:
+        schema = '{"queries": ["...", "..."]}'
+    lines.append(f"Respond with JSON matching this exact schema: {schema}")
+    return "\n".join(lines)
+
+
+def _extract_json(raw: str) -> dict:
+    text = raw.strip()
+    # strip markdown fences if present
+    if text.startswith("```"):
+        text = text.split("\n", 1)[1] if "\n" in text else text
+        if text.endswith("```"):
+            text = text.rsplit("```", 1)[0]
+    start = text.find("{")
+    end = text.rfind("}")
+    if start == -1 or end == -1 or end < start:
+        raise ValueError("no JSON object found")
+    return json.loads(text[start : end + 1])
+
+
+def expand_query(gen: Generator, query: str, *, variants: int = 3, hyde: bool = True) -> Expansion:
+    empty = Expansion([], None)
+    try:
+        # generous budget: reasoning models spend tokens thinking before the JSON
+        raw = gen.generate(_SYSTEM, _build_prompt(query, variants, hyde), max_tokens=4096)
+        data = _extract_json(raw)
+        if not isinstance(data, dict) or "queries" not in data:
+            raise ValueError(f"unexpected JSON shape: {data!r}")
+        queries = data["queries"]
+        if not isinstance(queries, list):
+            raise ValueError(f"'queries' is not a list: {queries!r}")
+    except Exception as exc:  # noqa: BLE001 - defensive per contract, never raise
+        print(f"warning: query expansion failed: {exc}", file=sys.stderr)
+        return empty
+
+    seen = {query.strip().lower()}
+    out: list[str] = []
+    for item in queries:
+        if not isinstance(item, str):
+            continue
+        cleaned = item.strip()
+        if not cleaned:
+            continue
+        key = cleaned.lower()
+        if key in seen:
+            continue
+        seen.add(key)
+        out.append(cleaned)
+        if len(out) >= variants:
+            break
+
+    hyde_text = data.get("hyde") if hyde and isinstance(data, dict) else None
+    if not isinstance(hyde_text, str) or not hyde_text.strip():
+        hyde_text = None
+    return Expansion(out, hyde_text)

ragx/core/fusion.py

@@ -0,0 +1,14 @@
+"""Reciprocal Rank Fusion over multiple rankings."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+
+def rrf(rankings: Sequence[Sequence[int]], k: int = 60) -> dict[int, float]:
+    """score(d) = sum over rankings containing d of 1/(k + rank), rank 1-based."""
+    scores: dict[int, float] = {}
+    for ranking in rankings:
+        for rank, doc_id in enumerate(ranking, start=1):
+            scores[doc_id] = scores.get(doc_id, 0.0) + 1.0 / (k + rank)
+    return scores

ragx/core/graph.py

@@ -0,0 +1,37 @@
+"""kNN edge construction over a VectorIndex.
+
+Pure functions: no Store access. Callers persist the resulting edges.
+"""
+
+from __future__ import annotations
+
+from typing import Sequence
+
+from ragx.core.vectors import VectorIndex
+
+
+def knn_edges(
+    index: VectorIndex, ids: Sequence[int], k: int, min_sim: float
+) -> dict[int, list[tuple[int, float]]]:
+    """For each id, search its own stored vector for k+1 hits, drop self,
+    keep sim >= min_sim, cap at k. Returns id -> [(neighbor_id, sim), ...] sim-desc.
+    """
+    if not ids:
+        return {}
+    vectors = index.get_vectors(ids)
+    result: dict[int, list[tuple[int, float]]] = {}
+    for chunk_id, vector in zip(ids, vectors):
+        hits = index.search(vector, k + 1)
+        neighbors = [
+            (nid, sim) for nid, sim in hits if nid != chunk_id and sim >= min_sim
+        ][:k]
+        result[chunk_id] = neighbors
+    return result
+
+
+def affected_ids(new_edges: dict[int, list[tuple[int, float]]]) -> set[int]:
+    """Every neighbor id referenced in new_edges values, minus new_edges' own keys —
+    the incremental-maintenance set whose edge lists the indexer must recompute.
+    """
+    referenced = {nid for neighbors in new_edges.values() for nid, _ in neighbors}
+    return referenced - new_edges.keys()

ragx/core/indexer.py

@@ -0,0 +1,118 @@
+"""Indexing pipeline: discover -> hash -> chunk -> embed -> vector index + store."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+
+from ragx.core.chunking import chunk_text
+from ragx.core.config import Config, db_path, vectors_path
+from ragx.core.discovery import discover_files, hash_file
+from ragx.core.errors import ManifestMismatchError
+from ragx.core.graph import affected_ids, knn_edges
+from ragx.core.models import FileRecord
+from ragx.core.store import Store
+from ragx.core.vectors import VectorIndex
+from ragx.providers.base import Embedder
+
+log = logging.getLogger("ragx.index")
+
+
+@dataclass
+class IndexStats:
+    files_indexed: int = 0
+    files_deleted: int = 0
+    files_unchanged: int = 0
+    chunks_added: int = 0
+    chunks_deleted: int = 0
+    edges_total: int = 0
+
+
+def run_index(root: Path, cfg: Config, embedder: Embedder, *, changed_only: bool = False) -> IndexStats:
+    """Index the corpus at `root`. Full rebuild by default; hash-diff when changed_only."""
+    with Store(db_path(root)) as store:
+        _check_manifest(store, embedder, allow_rewrite=not changed_only)
+        dim = embedder.dimension()
+        store.set_meta("embedding_model", embedder.model)
+        store.set_meta("embedding_dim", str(dim))
+
+        current = {
+            rel: hash_file(root / rel)
+            for rel in discover_files(
+                root,
+                cfg.get("corpus.include"),
+                cfg.get("corpus.exclude"),
+                cfg.get("corpus.respect_gitignore"),
+            )
+        }
+        known = store.get_file_hashes()
+
+        if changed_only:
+            to_index = [p for p, h in current.items() if known.get(p) != h]
+            to_delete = [p for p in known if p not in current]
+            index = _open_vectors(root, dim)
+        else:
+            to_index, to_delete = list(current), []
+            for p in known:
+                store.delete_file(p)
+            vectors_path(root).unlink(missing_ok=True)
+            index = _open_vectors(root, dim)
+
+        stats = IndexStats(files_unchanged=len(current) - len(to_index))
+
+        for path in to_delete:
+            removed = store.delete_file(path)
+            index.mark_deleted(removed)
+            stats.files_deleted += 1
+            stats.chunks_deleted += len(removed)
+
+        size_tokens = cfg.get("chunking.size_tokens")
+        overlap = cfg.get("chunking.overlap")
+        new_ids: list[int] = []
+        for path in sorted(to_index):
+            if path in known:  # changed file: drop old chunks first
+                removed = store.delete_file(path)
+                index.mark_deleted(removed)
+                stats.chunks_deleted += len(removed)
+            text = (root / path).read_text(encoding="utf-8", errors="replace")
+            drafts = chunk_text(text, path, size_tokens=size_tokens, overlap=overlap)
+            store.upsert_file(FileRecord(path, current[path], (root / path).stat().st_mtime, len(drafts)))
+            if not drafts:
+                stats.files_indexed += 1
+                continue
+            ids = store.insert_chunks(path, drafts)
+            vectors = embedder.embed_documents([d.text for d in drafts])
+            index.add(ids, vectors)
+            new_ids.extend(ids)
+            stats.files_indexed += 1
+            stats.chunks_added += len(ids)
+            log.info("indexed %s (%d chunks)", path, len(ids))
+
+        if new_ids:
+            k, min_sim = cfg.get("graph.k"), cfg.get("graph.min_edge_sim")
+            edges = knn_edges(index, new_ids, k, min_sim)
+            touched = affected_ids(edges)
+            if touched:  # incremental: refresh edge lists of pre-existing neighbors
+                edges.update(knn_edges(index, sorted(touched), k, min_sim))
+            for src, nbrs in edges.items():
+                store.replace_edges(src, nbrs)
+            log.info("graph: %d nodes re-edged", len(edges))
+
+        stats.edges_total = store.edge_count()
+        index.save()
+        return stats
+
+
+def _check_manifest(store: Store, embedder: Embedder, *, allow_rewrite: bool) -> None:
+    prev = store.get_meta("embedding_model")
+    if prev and prev != embedder.model and not allow_rewrite:
+        raise ManifestMismatchError(
+            f"index was built with {prev!r} but config says {embedder.model!r}; "
+            "run a full `ragx index` to rebuild"
+        )
+
+
+def _open_vectors(root: Path, dim: int) -> VectorIndex:
+    vp = vectors_path(root)
+    return VectorIndex.load(vp, dim) if vp.exists() else VectorIndex.create(vp, dim)

ragx/core/models.py

@@ -0,0 +1,67 @@
+"""Core data types shared across modules. Frozen where identity matters."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True)
+class ChunkDraft:
+    """A chunk produced by a chunker, before it has an id or embedding."""
+
+    text: str
+    byte_start: int
+    byte_end: int
+    line_start: int  # 1-based, inclusive
+    line_end: int  # 1-based, inclusive
+
+
+@dataclass(frozen=True)
+class Chunk:
+    """A stored chunk. `id` is the SQLite rowid and doubles as the HNSW label."""
+
+    id: int
+    file_path: str  # relative to corpus root, POSIX separators
+    text: str
+    byte_start: int
+    byte_end: int
+    line_start: int
+    line_end: int
+
+
+@dataclass(frozen=True)
+class FileRecord:
+    path: str  # relative to corpus root, POSIX separators
+    content_hash: str  # xxhash64 hexdigest
+    mtime: float
+    chunk_count: int
+
+
+@dataclass(frozen=True)
+class Edge:
+    """Undirected similarity edge. Stored normalized with src < dst."""
+
+    src: int
+    dst: int
+    weight: float  # cosine similarity in [0, 1]
+
+
+@dataclass
+class ScoredChunk:
+    """A query result candidate with its score breakdown."""
+
+    chunk: Chunk
+    score: float = 0.0
+    vector_score: float = 0.0  # fused RRF score (or raw cosine in no-fusion mode)
+    heat: float = 0.0
+    rerank_score: float = 0.0
+    explain: dict | None = None  # traversal trace: seeds, edge paths, weights
+
+
+@dataclass
+class QueryOutput:
+    """Top-level result of the query pipeline; serialized as ragx.query.v1."""
+
+    query: str
+    variants: list[str] = field(default_factory=list)
+    results: list[ScoredChunk] = field(default_factory=list)