codexlr8 0.0.2__tar.gz → 0.0.3__tar.gz

{codexlr8-0.0.2 → codexlr8-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codexlr8
-Version: 0.0.2
+Version: 0.0.3
 Summary: A codebase search engine for LLM coding agents
 Author-email: Sadig Akhund <sadigaxund@gmail.com>
 License: Apache-2.0
@@ -25,6 +25,8 @@ Requires-Dist: mcp>=1.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.0; extra == "dev"
+Provides-Extra: embeddings
+Requires-Dist: sentence-transformers>=3.0; extra == "embeddings"
 Dynamic: license-file
 
 # CodeXLR8
@@ -95,6 +97,26 @@ codexlr8 search . "axes not hiding" --explain
 # Combine both — group, then scope to drill down
 ```
 
+### Search Quality & Fine-Tuning
+
+```bash
+# Measure search accuracy against known queries
+codexlr8 eval . --queries queries.json
+# Precision@1: 67%, MRR: 0.83, Recall@5: 67%
+
+# Typos are auto-corrected (fuzzy fallback on zero results)
+codexlr8 search . "funtion"  # → corrects to "function"
+
+# Opt-in embeddings: hybrid BM25 + semantic search
+# pip install codexlr8[embeddings]
+# set embeddings.enabled: true in .codexlr8.yaml
+
+# Fine-tune a model on YOUR codebase vocabulary
+codexlr8 recommend-model .   # picks the right model for your size
+codexlr8 train .              # TSDAE training, 5-45min on CPU
+codexlr8 eval .               # measure improvement
+```
+
 ## .meta.yaml Sidecars
 
 Optional YAML files next to source files, created by `codexlr8 init`:

{codexlr8-0.0.2 → codexlr8-0.0.3}/README.md

@@ -66,6 +66,26 @@ codexlr8 search . "axes not hiding" --explain
 # Combine both — group, then scope to drill down
 ```
 
+### Search Quality & Fine-Tuning
+
+```bash
+# Measure search accuracy against known queries
+codexlr8 eval . --queries queries.json
+# Precision@1: 67%, MRR: 0.83, Recall@5: 67%
+
+# Typos are auto-corrected (fuzzy fallback on zero results)
+codexlr8 search . "funtion"  # → corrects to "function"
+
+# Opt-in embeddings: hybrid BM25 + semantic search
+# pip install codexlr8[embeddings]
+# set embeddings.enabled: true in .codexlr8.yaml
+
+# Fine-tune a model on YOUR codebase vocabulary
+codexlr8 recommend-model .   # picks the right model for your size
+codexlr8 train .              # TSDAE training, 5-45min on CPU
+codexlr8 eval .               # measure improvement
+```
+
 ## .meta.yaml Sidecars
 
 Optional YAML files next to source files, created by `codexlr8 init`:

{codexlr8-0.0.2 → codexlr8-0.0.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "codexlr8"
-version = "0.0.2"
+version = "0.0.3"
 description = "A codebase search engine for LLM coding agents"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -37,6 +37,9 @@ dev = [
     "pytest>=7.0",
     "pytest-cov>=4.0",
 ]
+embeddings = [
+    "sentence-transformers>=3.0",
+]
 
 [project.scripts]
 codexlr8 = "codexlr8.cli:main"

{codexlr8-0.0.2 → codexlr8-0.0.3}/src/codexlr8/__init__.py

@@ -1,3 +1,3 @@
 """CodeXLR8 — A codebase search engine for LLM coding agents."""
 
-__version__ = "0.0.2"
+__version__ = "0.0.3"

{codexlr8-0.0.2 → codexlr8-0.0.3}/src/codexlr8/cli.py

@@ -202,6 +202,140 @@ def status(project_path: str):
         click.secho(f"  Warning: {state['warning']}", fg="yellow")
 
 
+@main.command()
+@click.argument("project_path", type=click.Path(exists=True, file_okay=False))
+@click.option("--queries", "-q", required=True,
+              type=click.Path(exists=True, dir_okay=False),
+              help="Path to JSON file with query definitions")
+@click.option("--limit", "-n", default=10,
+              help="Max results per query (default: 10)")
+def eval_cmd(project_path: str, queries: str, limit: int):
+    """Evaluate search quality against a query set.
+
+    QUERIES is a JSON file with an array of query objects:
+    [{"query": "...", "expected": "path/to/file.py", "min_rank": 1}]
+
+    Outputs a per-query pass/fail table and aggregate metrics:
+    Precision@1, Mean Reciprocal Rank (MRR), Recall@5.
+    """
+    from .eval import load_queries, run_eval
+    import json
+
+    try:
+        query_defs = load_queries(queries)
+    except (json.JSONDecodeError, ValueError) as e:
+        raise click.ClickException(f"Invalid queries file: {e}")
+
+    if not query_defs:
+        raise click.ClickException("Queries file contains no queries.")
+
+    metrics = run_eval(project_path, query_defs, limit=limit)
+
+    # Per-query table
+    click.secho("  Query                             Expected            Mode   Lines    Rank  Score   Status", fg="cyan", bold=True)
+    click.secho("  " + "─" * 105, fg="cyan")
+
+    for r in metrics["results"]:
+        query_str = f'"{r["query"]}"'.ljust(34)
+        expected_str = r["expected"].ljust(20)
+        mode_str = r.get("assert", "file").ljust(7)
+        lines_str = ""
+        if r.get("line_start"):
+            lines_str = f"{r['line_start']}-{r['line_end']}".ljust(8)
+        else:
+            lines_str = "—".ljust(8)
+        rank_str = str(r["rank"]).ljust(6) if r["rank"] else "—     "
+        score_str = f'{r["score"]:.2f}'.ljust(8) if r["score"] else "—       "
+        status = r["status"]
+
+        if status.startswith("pass"):
+            status_style = {"fg": "green"}
+        elif "found" in status:
+            status_style = {"fg": "yellow"}
+        else:
+            status_style = {"fg": "red"}
+
+        click.echo(f"  {query_str} {expected_str} {mode_str} {lines_str} {rank_str} {score_str} {click.style(status, **status_style)}")
+
+    # Aggregate metrics
+    click.echo()
+    click.echo(click.style("  " + "─" * 40, fg="cyan"))
+    click.secho(f"  Precision@1:  {metrics['precision_at_1']:.2%}  "
+                f"({metrics['passed']}/{metrics['num_queries']} passed)", fg="green")
+    click.secho(f"  MRR:          {metrics['mrr']:.4f}", fg="green")
+    click.secho(f"  Recall@5:     {metrics['recall_at_5']:.2%}", fg="green")
+
+
+@main.command()
+@click.argument("project_path", type=click.Path(exists=True, file_okay=False), default=".")
+@click.option("--model", "-m", default="all-MiniLM-L6-v2",
+              help="Embedding model to fine-tune")
+@click.option("--epochs", "-e", default=3,
+              help="Training epochs (default: 3)")
+@click.option("--incremental", "-i", is_flag=True, default=False,
+              help="Fine-tune only on changed files")
+def train(project_path: str, model: str, epochs: int, incremental: bool):
+    """Fine-tune an embedding model on this codebase for better search accuracy.
+
+    Uses TSDAE (denoising auto-encoder) to adapt a pretrained model to
+    your codebase's vocabulary. The fine-tuned model is saved to
+    .codexlr8_model/ and referenced in .codexlr8.yaml.
+
+    Requirements: pip install codexlr8[embeddings]
+    """
+    try:
+        from .train import train_model
+    except ImportError as e:
+        raise click.ClickException(
+            "Training requires 'pip install codexlr8[embeddings]'"
+        ) from e
+
+    click.echo()
+    click.secho("  Training embedding model on this codebase...", fg="cyan", bold=True)
+    click.echo(f"  Model:  {model}")
+    click.echo(f"  Epochs: {epochs}")
+    click.echo()
+
+    try:
+        result = train_model(project_path, model_name=model,
+                             epochs=epochs, incremental=incremental)
+    except ValueError as e:
+        raise click.ClickException(str(e))
+
+    dur = result["duration_sec"]
+    dur_str = f"{dur}s" if dur < 60 else f"{dur // 60}m{dur % 60}s"
+
+    click.echo()
+    click.secho(f"  Trained on {result['num_examples']} files in {dur_str}", fg="green")
+    click.secho(f"  Model saved to {result['model_path']}", fg="green")
+    click.secho(f"  Embeddings enabled in .codexlr8.yaml", fg="green")
+    click.echo()
+    click.secho("  Run 'codexlr8 eval .' to measure improvement.", dim=True)
+
+
+@main.command()
+@click.argument("project_path", type=click.Path(exists=True, file_okay=False), default=".")
+def recommend_model_cmd(project_path: str):
+    """Suggest the best embedding model for this codebase size."""
+    try:
+        from .train import recommend_model
+    except ImportError as e:
+        raise click.ClickException(
+            "Requires 'pip install codexlr8[embeddings]'"
+        ) from e
+
+    rec = recommend_model(project_path)
+
+    click.echo()
+    click.secho(f"  Codebase: {rec['num_files']} files, ~{rec['est_tokens']:,} tokens", fg="cyan")
+    click.echo()
+    click.secho(f"  Recommended: {rec['model']} ({rec['param_count']})", fg="green", bold=True)
+    click.echo(f"  Est. training time: {rec['est_training_time']}")
+    click.echo(f"  Expected quality gain: {rec.get('quality_gain', '+5-12% MRR')}")
+    click.echo()
+    click.secho("  Run 'codexlr8 train .' to start training.", dim=True)
+
+
 @main.command()
 @click.argument("project_path", type=click.Path(exists=True, file_okay=False), default=".")
 def setup(project_path: str):
@@ -702,6 +836,8 @@ Exclude patterns are globs that match file paths. Use `*` for wildcards.
 | Search within a directory | `codebase_search(query="...", scope="src/")` |
 | Cluster results by directory | Shell: `codexlr8 search . "query" --grouped` |
 | Diagnose query terms | Shell: `codexlr8 search . "query" --explain` |
+| Measure search accuracy | Shell: `codexlr8 eval . --queries q.json` |
+| Fine-tune embeddings | Shell: `codexlr8 train .` (needs `[embeddings]` extra) |
 | Build/update index | `codebase_index(incremental=true)` |
 | Check metadata coverage | Shell: `codexlr8 status .` |
 | Bootstrap missing sidecars | Shell: `codexlr8 init .` |

{codexlr8-0.0.2 → codexlr8-0.0.3}/src/codexlr8/config.py

@@ -24,6 +24,12 @@ def load_config(project_path: str) -> dict:
 def _defaults() -> dict:
     return {
         "root": ".",
+        "fuzzy": True,
+        "embeddings": {
+            "enabled": False,
+            "model": "all-MiniLM-L6-v2",
+            "bm25_weight": 0.6,
+        },
         "include": [],
         "exclude": [
             "tests/*",

codexlr8-0.0.3/src/codexlr8/embeddings.py

@@ -0,0 +1,147 @@
+"""Embedding provider — optional semantic search layer.
+
+Requires optional dependencies: pip install codexlr8[embeddings]
+Provides cosine-similarity reranking on top of BM25 results.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import os
+import sqlite3
+
+_EMBEDDING_AVAILABLE = False
+_SentenceTransformer = None
+
+
+def _check_deps() -> bool:
+    """Lazy-import sentence-transformers. Returns True if available."""
+    global _EMBEDDING_AVAILABLE, _SentenceTransformer
+    if _EMBEDDING_AVAILABLE:
+        return True
+    try:
+        from sentence_transformers import SentenceTransformer as ST
+        _SentenceTransformer = ST
+        _EMBEDDING_AVAILABLE = True
+        return True
+    except ImportError:
+        return False
+
+
+class EmbeddingProvider:
+    """Lazy-loading sentence-transformers model for embedding text."""
+
+    def __init__(self, model_name: str = "all-MiniLM-L6-v2"):
+        self.model_name = model_name
+        self._model = None
+
+    @property
+    def model(self):
+        if self._model is None:
+            if not _check_deps():
+                raise ImportError(
+                    "Embeddings require 'pip install codexlr8[embeddings]' "
+                    "(installs sentence-transformers)"
+                )
+            self._model = _SentenceTransformer(self.model_name)
+        return self._model
+
+    @property
+    def dims(self) -> int:
+        return self.model.get_sentence_embedding_dimension()
+
+    def embed(self, texts: list[str], batch_size: int = 32) -> list[list[float]]:
+        """Encode texts into normalized embedding vectors."""
+        if not texts:
+            return []
+        embeddings = self.model.encode(
+            texts,
+            batch_size=batch_size,
+            normalize_embeddings=True,
+            show_progress_bar=False,
+        )
+        return embeddings.tolist()
+
+    def embed_file(self, path: str, content: str, summary: str = "",
+                   tags: list[str] | None = None) -> list[float]:
+        """Embed a single file combining path, summary, tags, and content preview."""
+        tags_str = " ".join(tags) if tags else ""
+        # Combine metadata with first 2000 chars of content for context
+        text = f"{path} {summary} {tags_str} {content[:2000]}"
+        return self.embed([text])[0]
+
+
+def cosine_similarity(a: list[float], b: list[float]) -> float:
+    """Compute cosine similarity between two normalized vectors."""
+    if not a or not b:
+        return 0.0
+    dot = sum(x * y for x, y in zip(a, b))
+    # Clamp to [-1, 1] for floating-point safety
+    return max(-1.0, min(1.0, dot))
+
+
+def init_embeddings_table(conn: sqlite3.Connection):
+    """Create the embeddings table if it doesn't exist."""
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS embeddings (
+            path TEXT PRIMARY KEY,
+            vector TEXT,
+            dims INTEGER,
+            embedded_at TEXT
+        )
+    """)
+
+
+def store_embeddings(conn: sqlite3.Connection, path: str,
+                     vector: list[float], embedded_at: str):
+    """Store or update an embedding vector for a file."""
+    conn.execute(
+        "INSERT OR REPLACE INTO embeddings (path, vector, dims, embedded_at) "
+        "VALUES (?, ?, ?, ?)",
+        (path, json.dumps(vector), len(vector), embedded_at),
+    )
+
+
+def load_embeddings(conn: sqlite3.Connection) -> dict[str, list[float]]:
+    """Load all stored embeddings from the database."""
+    rows = conn.execute("SELECT path, vector FROM embeddings").fetchall()
+    return {row["path"]: json.loads(row["vector"]) for row in rows}
+
+
+def hybrid_rerank(
+    bm25_results: list[dict],
+    embedded_vectors: dict[str, list[float]],
+    query_vector: list[float],
+    bm25_weight: float = 0.6,
+    embed_weight: float = 0.4,
+) -> list[dict]:
+    """Merge BM25 scores with cosine similarity and re-rank results.
+
+    bm25_weight / embed_weight control the blending.
+    Default 0.6/0.4 favors BM25 (precise token matching) with semantic uplift.
+    """
+    if not query_vector or not embedded_vectors:
+        return bm25_results
+
+    # Normalize BM25 scores to [0, 1] range
+    scores = [r["score"] for r in bm25_results]
+    if not scores:
+        return bm25_results
+    max_score = max(scores)
+    min_score = min(scores)
+    score_range = max_score - min_score if max_score != min_score else 1.0
+
+    for r in bm25_results:
+        bm25_norm = (r["score"] - min_score) / score_range
+
+        vec = embedded_vectors.get(r["path"])
+        cosine = cosine_similarity(query_vector, vec) if vec else 0.0
+
+        # Weighted blend
+        r["score"] = round(bm25_weight * bm25_norm + embed_weight * cosine, 4)
+        r["_bm25_norm"] = round(bm25_norm, 4)
+        r["_cosine"] = round(cosine, 4)
+
+    bm25_results.sort(key=lambda r: r["score"], reverse=True)
+    return bm25_results

codexlr8-0.0.3/src/codexlr8/eval.py

@@ -0,0 +1,246 @@
+"""Search quality evaluation — measure query-to-result accuracy."""
+
+from __future__ import annotations
+
+import json
+import os
+
+from .search import SearchEngine
+
+
+def load_queries(path: str) -> list[dict]:
+    """Load evaluation queries from a JSON file.
+
+    Schema:
+    [
+      {"query": "login auth", "expected": "auth/session.py", "min_rank": 1},
+      {"query": "login auth", "expected": "auth/session.py",
+       "scope": {"start": 14, "end": 27}, "assert": "scope"}
+    ]
+
+    Fields:
+      query    — search query string
+      expected — file path that should appear in results
+      min_rank — required ranking position (default 1)
+      scope    — line range the result should cover: {start, end}
+      assert   — "file" (default), "scope", or "exact"
+    """
+    with open(path, "r", encoding="utf-8") as f:
+        queries = json.load(f)
+
+    if not isinstance(queries, list):
+        raise ValueError("Queries file must be a JSON array")
+
+    for i, q in enumerate(queries):
+        for key in ("query", "expected"):
+            if key not in q:
+                raise ValueError(f"Query item {i}: missing required key '{key}'")
+        q.setdefault("min_rank", 1)
+        q.setdefault("assert", "file")
+        if q["assert"] not in ("file", "scope", "exact"):
+            raise ValueError(
+                f"Query item {i}: assert must be 'file', 'scope', or 'exact'"
+            )
+        if q["assert"] in ("scope", "exact") and "scope" not in q:
+            raise ValueError(
+                f"Query item {i}: assert='{q['assert']}' requires a 'scope' field"
+            )
+
+    return queries
+
+
+def run_eval(project_path: str, queries: list[dict],
+             limit: int = 10, exclude: list[str] | None = None,
+             scope: str | None = None) -> dict:
+    """Run search evaluation and return per-query results + aggregate metrics.
+
+    Returns: {
+        "project_path": str,
+        "num_queries": int,
+        "results": [{query, expected, rank, score, status, mode}],
+        "precision_at_1": float,
+        "recall_at_5": float,
+        "mrr": float,
+        "passed": int,
+        "failed": int,
+    }
+    """
+    engine = SearchEngine(project_path)
+
+    query_results = []
+    for q in queries:
+        result = _eval_one(engine, q, limit, exclude, scope)
+        query_results.append(result)
+
+    metrics = _compute_metrics(query_results)
+    metrics["project_path"] = project_path
+    metrics["num_queries"] = len(queries)
+    metrics["results"] = query_results
+
+    return metrics
+
+
+def _eval_one(engine: SearchEngine, query_def: dict,
+              limit: int, exclude: list[str] | None,
+              search_scope: str | None) -> dict:
+    """Run a single query and check if expected file/scope appears in results."""
+    search_results = engine.search(
+        query_def["query"], limit=limit, exclude=exclude, scope=search_scope
+    )
+
+    expected_file = query_def["expected"]
+    min_rank = query_def.get("min_rank", 1)
+    assert_mode = query_def.get("assert", "file")
+    expected_scope = query_def.get("scope")
+
+    found = False
+    rank = None
+    score = None
+    matched = []
+    result_lines = (0, 0)
+
+    for i, r in enumerate(search_results):
+        if r["path"] == expected_file:
+            found = True
+            rank = i + 1
+            score = r["score"]
+            matched = r.get("matched_tokens", [])
+            result_lines = (r.get("line_start", 0), r.get("line_end", 0))
+            break
+
+    # File-level check
+    if not found:
+        return {
+            "query": query_def["query"],
+            "expected": expected_file,
+            "assert": assert_mode,
+            "scope": expected_scope,
+            "min_rank": min_rank,
+            "rank": None,
+            "score": None,
+            "matched_tokens": [],
+            "line_start": 0,
+            "line_end": 0,
+            "status": "fail",
+        }
+
+    if found and rank > min_rank:
+        return {
+            "query": query_def["query"],
+            "expected": expected_file,
+            "assert": assert_mode,
+            "scope": expected_scope,
+            "min_rank": min_rank,
+            "rank": rank,
+            "score": score,
+            "matched_tokens": matched,
+            "line_start": result_lines[0],
+            "line_end": result_lines[1],
+            "status": f"found@{rank} (needed ≤{min_rank})",
+        }
+
+    # File found at correct rank. Check scope if required.
+    scope_status = None
+    if assert_mode in ("scope", "exact") and expected_scope:
+        scope_status = _check_scope_overlap(result_lines, expected_scope, assert_mode)
+
+    if scope_status:
+        return {
+            "query": query_def["query"],
+            "expected": expected_file,
+            "assert": assert_mode,
+            "scope": expected_scope,
+            "min_rank": min_rank,
+            "rank": rank,
+            "score": score,
+            "matched_tokens": matched,
+            "line_start": result_lines[0],
+            "line_end": result_lines[1],
+            "status": scope_status,
+        }
+
+    # File-level pass
+    suffix = f" (top-{min_rank})" if min_rank > 1 else ""
+    return {
+        "query": query_def["query"],
+        "expected": expected_file,
+        "assert": assert_mode,
+        "scope": expected_scope,
+        "min_rank": min_rank,
+        "rank": rank,
+        "score": score,
+        "matched_tokens": matched,
+        "line_start": result_lines[0],
+        "line_end": result_lines[1],
+        "status": f"pass{suffix}",
+    }
+
+
+def _check_scope_overlap(
+    result_lines: tuple[int, int],
+    expected_scope: dict,
+    mode: str,
+) -> str | None:
+    """Check if result line range overlaps expected scope. Returns status or None."""
+    r_start, r_end = result_lines
+    e_start = expected_scope.get("start", 0)
+    e_end = expected_scope.get("end", 0)
+
+    if r_start == 0 or e_start == 0:
+        return "fail (no line data)"
+
+    overlap_start = max(r_start, e_start)
+    overlap_end = min(r_end, e_end)
+
+    if overlap_end <= overlap_start:
+        return "fail (no scope overlap)"
+
+    overlap_lines = overlap_end - overlap_start
+    expected_lines = e_end - e_start
+    ratio = overlap_lines / expected_lines if expected_lines > 0 else 0
+
+    if mode == "exact":
+        if ratio >= 0.8:
+            return f"pass (scope {ratio:.0%})"
+        return f"found (scope {ratio:.0%} < 80%)"
+    elif mode == "scope":
+        return "pass (scope overlap)"
+
+    return None
+
+
+def _compute_metrics(query_results: list[dict]) -> dict:
+    """Compute Precision@1, MRR, Recall@5 from per-query results."""
+    n = len(query_results)
+    if n == 0:
+        return {
+            "precision_at_1": 0.0,
+            "recall_at_5": 0.0,
+            "mrr": 0.0,
+            "passed": 0,
+            "failed": 0,
+        }
+
+    p1_count = 0
+    recall5_count = 0
+    reciprocal_sum = 0.0
+    passed = 0
+
+    for r in query_results:
+        rank = r["rank"]
+        if rank == 1:
+            p1_count += 1
+        if rank is not None and rank <= 5:
+            recall5_count += 1
+        if rank is not None:
+            reciprocal_sum += 1.0 / rank
+        if r["status"].startswith("pass"):
+            passed += 1
+
+    return {
+        "precision_at_1": round(p1_count / n, 4),
+        "recall_at_5": round(recall5_count / n, 4),
+        "mrr": round(reciprocal_sum / n, 4),
+        "passed": passed,
+        "failed": n - passed,
+    }