ragx-cli 0.1.0__py3-none-any.whl

ragx/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

ragx/cli/app.py

@@ -0,0 +1,105 @@
+"""ragx CLI shell: init, status, config. index/query are wired in a later pass."""
+
+from __future__ import annotations
+
+import logging
+import sys
+from pathlib import Path
+
+import typer
+
+from ragx.cli.output import emit_json, fail
+from ragx.core.config import Config, RAGX_DIR, db_path, require_root, write_default_config
+from ragx.core.errors import RagxError
+
+app = typer.Typer(add_completion=False, no_args_is_help=True)
+config_app = typer.Typer(add_completion=False, no_args_is_help=True)
+app.add_typer(config_app, name="config")
+
+from ragx.cli import eval_cmd, inspect_cmd, pipeline  # noqa: E402  (needs `app` defined above)
+
+pipeline.register(app)
+inspect_cmd.register(app)
+eval_cmd.register(app)
+
+
+def _require_root() -> Path:
+    try:
+        return require_root()
+    except RagxError as exc:
+        fail(str(exc))
+
+
+@app.command()
+def init(path: Path = typer.Argument(Path("."))) -> None:
+    """Create .ragx/ with a default config at PATH (default: cwd)."""
+    root = path.resolve()
+    if (root / RAGX_DIR).exists():
+        fail(f"{root / RAGX_DIR} already exists")
+    cfg_path = write_default_config(root)
+    typer.echo(f"created {cfg_path}")
+
+
+def _counts(root: Path) -> tuple[int, int, int]:
+    db = db_path(root)
+    if not db.exists():
+        return 0, 0, 0
+    try:
+        from ragx.core.store import Store
+    except ImportError:
+        return 0, 0, 0
+    with Store(db) as store:
+        return store.file_count(), store.chunk_count(), store.edge_count()
+
+
+@app.command()
+def status(json_out: bool = typer.Option(False, "--json")) -> None:
+    """Show corpus root, embedding config, and index counts."""
+    root = _require_root()
+    cfg = Config.load(root)
+    files, chunks, edges = _counts(root)
+    doc = {
+        "schema": "ragx.status.v1",
+        "root": str(root),
+        "embedding_provider": cfg.get("embeddings.provider"),
+        "embedding_model": cfg.get("embeddings.model"),
+        "files": files,
+        "chunks": chunks,
+        "edges": edges,
+    }
+    if json_out:
+        emit_json(doc)
+    else:
+        typer.echo(f"root: {doc['root']}")
+        typer.echo(f"embedding: {doc['embedding_provider']}/{doc['embedding_model']}")
+        typer.echo(f"files: {files}  chunks: {chunks}  edges: {edges}")
+    if files == 0 and chunks == 0 and edges == 0:
+        raise typer.Exit(code=1)
+
+
+@config_app.command("get")
+def config_get(key: str) -> None:
+    root = _require_root()
+    cfg = Config.load(root)
+    try:
+        value = cfg.get(key)
+    except RagxError as exc:
+        fail(str(exc))
+    typer.echo(str(value))
+
+
+@config_app.command("set")
+def config_set(key: str, value: str) -> None:
+    root = _require_root()
+    cfg = Config.load(root)
+    try:
+        cfg.set(key, value)
+    except RagxError as exc:
+        fail(str(exc))
+    cfg.save(root)
+    typer.echo(f"{key} = {cfg.get(key)}")
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO, stream=sys.stderr, format="%(levelname)s: %(message)s")
+    app()

ragx/cli/eval_cmd.py

@@ -0,0 +1,74 @@
+"""`ragx eval` command. Registered onto the main app by app.py (integration-owned)."""
+
+from __future__ import annotations
+
+from dataclasses import replace
+from pathlib import Path
+
+import typer
+
+from ragx.cli.output import emit_json, fail
+from ragx.core.config import Config, require_root
+from ragx.core.errors import RagxError
+from ragx.core.eval import evaluate, load_queries
+from ragx.core.query import QueryOptions, run_query
+from ragx.providers.registry import make_embedder, make_generator, make_reranker
+
+_BUILTIN_CONFIGS: dict[str, QueryOptions] = {
+    "baseline": QueryOptions(expand=False, graph=False, rerank=False),
+    "graph": QueryOptions(expand=False, graph=True, rerank=False),
+    "rerank": QueryOptions(expand=False, graph=True, rerank=True),
+    "full": QueryOptions(expand=True, graph=True, rerank=True),
+}
+
+
+def register(app: typer.Typer) -> None:
+    app.command("eval")(eval_cmd)
+
+
+def eval_cmd(
+    queries_file: Path = typer.Argument(..., help="path to a queries.jsonl file"),
+    json_out: bool = typer.Option(False, "--json"),
+    top: int = typer.Option(10, "--top"),
+    configs: str = typer.Option("baseline,graph,full", "--configs"),
+) -> None:
+    """Evaluate retrieval quality (recall@5/@10, MRR) against labeled queries."""
+    try:
+        queries = load_queries(queries_file)
+    except RagxError as exc:
+        fail(str(exc))
+
+    names = [n.strip() for n in configs.split(",") if n.strip()]
+    unknown = [n for n in names if n not in _BUILTIN_CONFIGS]
+    if unknown:
+        fail(f"unknown config(s): {', '.join(unknown)}")
+
+    try:
+        root = require_root()
+        cfg = Config.load(root)
+        embedder = make_embedder(cfg)
+        generator = make_generator(cfg) if any(_BUILTIN_CONFIGS[n].expand for n in names) else None
+        reranker = make_reranker(cfg) if any(_BUILTIN_CONFIGS[n].rerank for n in names) else None
+    except RagxError as exc:
+        fail(str(exc))
+
+    def query_fn(text: str, opts: QueryOptions):
+        return run_query(
+            root, cfg, embedder, text, opts,
+            generator=generator if opts.expand else None,
+            reranker=reranker if opts.rerank else None,
+        )
+
+    # retrieve chunks deep enough that file-level ranking has >= `top` files to rank
+    selected = [
+        (name, replace(_BUILTIN_CONFIGS[name], top=top * 3)) for name in names
+    ]
+    result = evaluate(queries, selected, query_fn, top=top)
+
+    if json_out:
+        emit_json(result)
+        return
+    header = f"{'config':<12}{'recall@5':>10}{'recall@10':>10}{'mrr':>10}"
+    typer.echo(header)
+    for c in result["configs"]:
+        typer.echo(f"{c['name']:<12}{c['recall_at_5']:>10.4f}{c['recall_at_10']:>10.4f}{c['mrr']:>10.4f}")

ragx/cli/inspect_cmd.py

@@ -0,0 +1,122 @@
+"""ragx inspect: read-only introspection into the store (chunks, files, neighbors).
+
+No embeddings or vector index needed — everything here reads Store directly.
+"""
+
+from __future__ import annotations
+
+import typer
+
+from ragx.cli.output import emit_json, fail
+from ragx.core.config import db_path, require_root
+from ragx.core.errors import RagxError
+from ragx.core.store import Store
+
+inspect_app = typer.Typer(add_completion=False, no_args_is_help=True)
+
+
+def register(app: typer.Typer) -> None:
+    app.add_typer(inspect_app, name="inspect")
+
+
+def _open_store() -> Store:
+    try:
+        root = require_root()
+    except RagxError as exc:
+        fail(str(exc))
+    return Store(db_path(root))
+
+
+def _neighbor_info(store: Store, chunk_id: int) -> list[dict]:
+    raw = store.neighbors(chunk_id)
+    others = {c.id: c for c in store.get_chunks([other_id for other_id, _ in raw])}
+    return [
+        {
+            "id": other_id,
+            "weight": round(weight, 6),
+            "file": others[other_id].file_path if other_id in others else None,
+            "line_start": others[other_id].line_start if other_id in others else None,
+            "line_end": others[other_id].line_end if other_id in others else None,
+        }
+        for other_id, weight in raw
+    ]
+
+
+@inspect_app.command("chunk")
+def inspect_chunk(chunk_id: int, json_out: bool = typer.Option(False, "--json")) -> None:
+    """Show a chunk's text, location, and edges."""
+    with _open_store() as store:
+        chunks = store.get_chunks([chunk_id])
+        if not chunks:
+            fail(f"unknown chunk id: {chunk_id}")
+        chunk = chunks[0]
+        edges = _neighbor_info(store, chunk_id)
+    doc = {
+        "schema": "ragx.inspect.chunk.v1",
+        "id": chunk.id,
+        "file": chunk.file_path,
+        "byte_start": chunk.byte_start,
+        "byte_end": chunk.byte_end,
+        "line_start": chunk.line_start,
+        "line_end": chunk.line_end,
+        "text": chunk.text,
+        "edges": edges,
+    }
+    if json_out:
+        emit_json(doc)
+    else:
+        typer.echo(f"chunk {chunk.id}  {chunk.file_path}:{chunk.line_start}-{chunk.line_end}")
+        typer.echo(chunk.text)
+        typer.echo(f"edges ({len(edges)}):")
+        for e in edges:
+            typer.echo(f"  {e['id']}  {e['weight']:.4f}  {e['file']}")
+    if not edges:
+        raise typer.Exit(code=1)
+
+
+@inspect_app.command("file")
+def inspect_file(path: str, json_out: bool = typer.Option(False, "--json")) -> None:
+    """Show a file's record and its chunks."""
+    with _open_store() as store:
+        hashes = store.get_file_hashes()
+        if path not in hashes:
+            fail(f"unknown file: {path}")
+        ids = store.chunk_ids_for_file(path)
+        chunks = store.get_chunks(ids)
+    chunk_docs = [
+        {"id": c.id, "line_start": c.line_start, "line_end": c.line_end, "preview": c.text[:80]}
+        for c in chunks
+    ]
+    doc = {
+        "schema": "ragx.inspect.file.v1",
+        "path": path,
+        "content_hash": hashes[path],
+        "chunk_count": len(chunk_docs),
+        "chunks": chunk_docs,
+    }
+    if json_out:
+        emit_json(doc)
+    else:
+        typer.echo(f"file {path}  ({doc['chunk_count']} chunks)  hash={doc['content_hash']}")
+        for c in chunk_docs:
+            typer.echo(f"  {c['id']}  {c['line_start']}-{c['line_end']}  {c['preview']}")
+    if not chunk_docs:
+        raise typer.Exit(code=1)
+
+
+@inspect_app.command("neighbors")
+def inspect_neighbors(chunk_id: int, json_out: bool = typer.Option(False, "--json")) -> None:
+    """Show a chunk's neighbors enriched with file/line info."""
+    with _open_store() as store:
+        if not store.get_chunks([chunk_id]):
+            fail(f"unknown chunk id: {chunk_id}")
+        neighbors = _neighbor_info(store, chunk_id)
+    doc = {"schema": "ragx.inspect.neighbors.v1", "id": chunk_id, "neighbors": neighbors}
+    if json_out:
+        emit_json(doc)
+    else:
+        typer.echo(f"neighbors of {chunk_id}:")
+        for n in neighbors:
+            typer.echo(f"  {n['id']}  {n['weight']:.4f}  {n['file']}:{n['line_start']}-{n['line_end']}")
+    if not neighbors:
+        raise typer.Exit(code=1)

ragx/cli/output.py

@@ -0,0 +1,19 @@
+"""Stdout/exit conventions for the CLI. stdout is reserved for result payloads;
+everything diagnostic goes to stderr (configured in app.main())."""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import NoReturn
+
+
+def emit_json(doc: dict) -> None:
+    """Write exactly one JSON document to stdout, nothing else on stdout."""
+    sys.stdout.write(json.dumps(doc) + "\n")
+
+
+def fail(msg: str, code: int = 2) -> NoReturn:
+    """Print an error message to stderr and exit with `code`."""
+    print(msg, file=sys.stderr)
+    raise SystemExit(code)

ragx/cli/pipeline.py

@@ -0,0 +1,93 @@
+"""`index` and `query` commands. Registered onto the main app by app.py."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import typer
+
+from ragx.cli.output import emit_json, fail
+from ragx.core.config import Config, require_root
+from ragx.core.errors import RagxError
+from ragx.core.indexer import run_index
+from ragx.core.query import QueryOptions, run_query, to_files_json, to_query_json
+from ragx.providers.registry import make_embedder, make_generator, make_reranker
+
+
+def register(app: typer.Typer) -> None:
+    app.command()(index)
+    app.command()(query)
+
+
+def index(
+    path: Path | None = typer.Argument(None),
+    changed: bool = typer.Option(False, "--changed", help="only re-process new/modified/deleted files"),
+    json_out: bool = typer.Option(False, "--json"),
+) -> None:
+    """Chunk, embed, and index the corpus (full rebuild unless --changed)."""
+    try:
+        root = require_root(path)
+        cfg = Config.load(root)
+        stats = run_index(root, cfg, make_embedder(cfg), changed_only=changed)
+    except RagxError as exc:
+        fail(str(exc))
+    doc = {"schema": "ragx.index.v1", **stats.__dict__}
+    if json_out:
+        emit_json(doc)
+    else:
+        typer.echo(
+            f"indexed {stats.files_indexed} files ({stats.chunks_added} chunks), "
+            f"deleted {stats.files_deleted} ({stats.chunks_deleted} chunks), "
+            f"unchanged {stats.files_unchanged}"
+        )
+
+
+def query(
+    text: str = typer.Argument(..., help="query text, or '-' to read from stdin"),
+    top: int | None = typer.Option(None, "--top"),
+    json_out: bool = typer.Option(False, "--json"),
+    files_only: bool = typer.Option(False, "--files-only"),
+    no_expand: bool = typer.Option(False, "--no-expand"),
+    no_graph: bool = typer.Option(False, "--no-graph"),
+    no_rerank: bool = typer.Option(False, "--no-rerank"),
+    hops: int | None = typer.Option(None, "--hops"),
+    explain: bool = typer.Option(False, "--explain"),
+) -> None:
+    """Search the index; ranked chunks (or files with --files-only)."""
+    if text == "-":
+        text = sys.stdin.read().strip()
+    try:
+        root = require_root()
+        cfg = Config.load(root)
+        opts = QueryOptions(
+            top=top if top is not None else cfg.get("query.top"),
+            files_only=files_only,
+            expand=not no_expand,
+            graph=not no_graph,
+            rerank=not no_rerank,
+            hops=hops,
+            explain=explain,
+        )
+        out = run_query(
+            root, cfg, make_embedder(cfg), text, opts,
+            generator=make_generator(cfg) if opts.expand else None,
+            reranker=make_reranker(cfg) if opts.rerank else None,
+        )
+    except RagxError as exc:
+        fail(str(exc))
+    if json_out:
+        if files_only:
+            emit_json(to_files_json(out))
+        else:
+            emit_json(to_query_json(out, max_chunk_chars=cfg.get("query.max_chunk_chars")))
+    elif files_only:
+        for f in to_files_json(out)["files"]:
+            typer.echo(f"{f['score']:.4f}  {f['file']}")
+    else:
+        for r in out.results:
+            loc = f"{r.chunk.file_path}:{r.chunk.line_start}-{r.chunk.line_end}"
+            first_line = r.chunk.text.strip().splitlines()[0][:100] if r.chunk.text.strip() else ""
+            typer.echo(f"{r.score:.4f}  {loc}  {first_line}")
+    if not out.results:
+        raise typer.Exit(code=1)

ragx/core/chunking.py

@@ -0,0 +1,183 @@
+"""Text chunking: markdown/code/recursive splitters producing byte-exact ChunkDraft slices."""
+
+from __future__ import annotations
+
+import re
+from bisect import bisect_right
+from pathlib import Path
+
+from ragx.core.models import ChunkDraft
+
+MARKDOWN_EXTS = {".md", ".markdown"}
+CODE_EXTS = {".py", ".ts", ".js", ".tsx", ".jsx", ".go", ".rs", ".java", ".rb"}
+SEPARATORS = ["\n\n", "\n", ". ", " "]
+MIN_CHUNK_CHARS = 100  # fragments below this merge into a neighbor (may nudge it past hard max)
+
+_HEADING_RE = re.compile(r"^#{1,6} .*$", re.MULTILINE)
+_CODE_BOUNDARY_RE = re.compile(r"^(?:def|class|function|fn|func)\b", re.MULTILINE)
+
+
+def _find_boundary(s: str, start: int, hard_end: int) -> int:
+    region = s[start:hard_end]
+    for sep in SEPARATORS:
+        idx = region.rfind(sep)
+        if idx != -1:
+            candidate = start + idx + len(sep)
+            if candidate > start:
+                return candidate
+    return hard_end
+
+
+def _recursive_split_range(s: str, target_chars: int, hard_max_chars: int, overlap: float) -> list[tuple[int, int]]:
+    n = len(s)
+    if n == 0:
+        return []
+    ranges: list[tuple[int, int]] = []
+    pos = 0
+    while pos < n:
+        remaining = n - pos
+        if remaining <= hard_max_chars:
+            end = n
+        else:
+            hard_end = min(pos + hard_max_chars, n)
+            end = _find_boundary(s, pos, hard_end)
+        ranges.append((pos, end))
+        if end >= n:
+            break
+        chunk_len = end - pos
+        overlap_chars = int(round(chunk_len * overlap))
+        next_pos = end - overlap_chars
+        if next_pos <= pos:
+            next_pos = end
+        pos = next_pos
+    return ranges
+
+
+def _recursive_split(
+    text: str, start: int, end: int, target_chars: int, hard_max_chars: int, overlap: float
+) -> list[tuple[int, int]]:
+    rel = _recursive_split_range(text[start:end], target_chars, hard_max_chars, overlap)
+    return [(start + a, start + b) for a, b in rel]
+
+
+def _sections_from_boundaries(text: str, starts: list[int]) -> list[tuple[int, int]]:
+    sections: list[tuple[int, int]] = []
+    if not starts:
+        return [(0, len(text))]
+    if starts[0] > 0:
+        sections.append((0, starts[0]))
+    for i, s in enumerate(starts):
+        e = starts[i + 1] if i + 1 < len(starts) else len(text)
+        sections.append((s, e))
+    return sections
+
+
+def _markdown_split(text: str, target_chars: int, hard_max_chars: int, overlap: float) -> list[tuple[int, int]]:
+    starts = [m.start() for m in _HEADING_RE.finditer(text)]
+    sections = _sections_from_boundaries(text, starts)
+
+    ranges: list[tuple[int, int]] = []
+    buffer: tuple[int, int] | None = None
+    for s, e in sections:
+        size = e - s
+        if size > hard_max_chars:
+            if buffer is not None:
+                ranges.append(buffer)
+                buffer = None
+            ranges.extend(_recursive_split(text, s, e, target_chars, hard_max_chars, overlap))
+            continue
+        if buffer is None:
+            buffer = (s, e)
+        elif (buffer[1] - buffer[0]) + size <= target_chars:
+            buffer = (buffer[0], e)
+        else:
+            ranges.append(buffer)
+            buffer = (s, e)
+    if buffer is not None:
+        ranges.append(buffer)
+    return ranges
+
+
+def _code_split(text: str, target_chars: int, hard_max_chars: int, overlap: float) -> list[tuple[int, int]]:
+    starts = [m.start() for m in _CODE_BOUNDARY_RE.finditer(text)]
+    sections = _sections_from_boundaries(text, starts)
+
+    ranges: list[tuple[int, int]] = []
+    for s, e in sections:
+        if (e - s) > hard_max_chars:
+            ranges.extend(_recursive_split(text, s, e, target_chars, hard_max_chars, overlap))
+        else:
+            ranges.append((s, e))
+    return ranges
+
+
+def _char_byte_offsets(text: str) -> list[int]:
+    offsets = [0] * (len(text) + 1)
+    total = 0
+    for i, ch in enumerate(text):
+        offsets[i] = total
+        total += len(ch.encode("utf-8"))
+    offsets[len(text)] = total
+    return offsets
+
+
+def _line_starts(text: str) -> list[int]:
+    starts = [0]
+    for i, ch in enumerate(text):
+        if ch == "\n":
+            starts.append(i + 1)
+    return starts
+
+
+def _merge_tiny_ranges(text: str, ranges: list[tuple[int, int]], min_chars: int) -> list[tuple[int, int]]:
+    """Absorb ranges with < min_chars of stripped text into the previous (or next) range."""
+    out: list[tuple[int, int]] = []
+    for start, end in ranges:
+        if out and len(text[start:end].strip()) < min_chars:
+            out[-1] = (out[-1][0], max(out[-1][1], end))
+        else:
+            out.append((start, end))
+    if len(out) >= 2 and len(text[out[0][0] : out[0][1]].strip()) < min_chars:
+        out[1] = (out[0][0], out[1][1])
+        out.pop(0)
+    return out
+
+
+def chunk_text(text: str, path: str, size_tokens: int = 800, overlap: float = 0.15) -> list[ChunkDraft]:
+    """Split `text` into byte-exact ChunkDrafts, dispatching on `path`'s extension."""
+    if not text.strip():
+        return []
+
+    target_chars = size_tokens * 4
+    hard_max_chars = int(round(target_chars * 1.5))
+
+    ext = Path(path).suffix.lower()
+    if ext in MARKDOWN_EXTS:
+        ranges = _markdown_split(text, target_chars, hard_max_chars, overlap)
+    elif ext in CODE_EXTS:
+        ranges = _code_split(text, target_chars, hard_max_chars, overlap)
+    else:
+        ranges = _recursive_split(text, 0, len(text), target_chars, hard_max_chars, overlap)
+    min_chars = min(MIN_CHUNK_CHARS, max(1, target_chars // 8))
+    ranges = _merge_tiny_ranges(text, ranges, min_chars)
+
+    char_to_byte = _char_byte_offsets(text)
+    line_starts = _line_starts(text)
+
+    drafts: list[ChunkDraft] = []
+    for start, end in ranges:
+        if end <= start:
+            continue
+        chunk_str = text[start:end]
+        if not chunk_str.strip():
+            continue
+        drafts.append(
+            ChunkDraft(
+                text=chunk_str,
+                byte_start=char_to_byte[start],
+                byte_end=char_to_byte[end],
+                line_start=bisect_right(line_starts, start),
+                line_end=bisect_right(line_starts, end - 1),
+            )
+        )
+    return drafts