codebase-receipts-cli 1.0.0__py3-none-any.whl

receipts/verify/kb.py

@@ -0,0 +1,379 @@
+"""Chroma-backed knowledge base over ingested code artifacts.
+
+Artifacts from the extractor are already individual functions, classes, or
+file excerpts — each one becomes a single Chroma document. Embeddings come
+from whatever LLM provider the user configured (Ollama, Gemini, etc.),
+keeping the KB module provider-agnostic.
+
+Persistence lives under ``~/.receipts/kb/<project>/`` so scanned repos are
+never polluted with index files.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import os
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+
+try:
+    import chromadb
+except (ImportError, OSError) as _chroma_err:
+    import platform as _platform
+    import sys as _sys
+
+    _arch = _platform.machine()
+    _os = _platform.system()
+    _py = _sys.version.split()[0]
+    raise SystemExit(
+        f"\n[receipts] chromadb failed to load on {_os}/{_arch} (Python {_py}).\n"
+        "\n"
+        "chromadb bundles hnswlib, a C++ extension. If no prebuilt wheel\n"
+        "exists for your architecture, pip needs to compile it from source.\n"
+        "\n"
+        "  Linux/ARM64:\n"
+        "    sudo apt install build-essential python3-dev\n"
+        "    pip install chromadb\n"
+        "\n"
+        "  macOS:\n"
+        "    xcode-select --install && pip install chromadb\n"
+        "\n"
+        "  Windows:\n"
+        "    Install Visual Studio Build Tools:\n"
+        "    https://visualstudio.microsoft.com/visual-cpp-build-tools/\n"
+        "    then: pip install chromadb\n"
+        "\n"
+        "If you are on an unsupported or unusual architecture (e.g. 32-bit x86,\n"
+        "RISC-V), chromadb may not be buildable. Open an issue and we can\n"
+        "evaluate a fallback vector store:\n"
+        "https://github.com/Ishaan-1606/receipts-cli/issues\n"
+        "\n"
+        f"Original error: {_chroma_err}\n"
+    ) from None
+
+from receipts.ingest.artifact_extractor import Artifact
+from receipts.llm.provider import LLMProvider
+from receipts.verify.bm25 import BM25Index, rrf_fuse
+
+_EMBED_BATCH_SIZE = 32
+
+
+def _hybrid_enabled() -> bool:
+    """Hybrid (dense + BM25) retrieval toggle — on unless RECEIPTS_HYBRID=0."""
+    return os.environ.get("RECEIPTS_HYBRID", "1").strip() != "0"
+
+
+def _cosine_distance(a: list[float], b: list[float]) -> float:
+    dot = sum(x * y for x, y in zip(a, b, strict=False))
+    norm_a = math.sqrt(sum(x * x for x in a))
+    norm_b = math.sqrt(sum(x * x for x in b))
+    if not norm_a or not norm_b:
+        return 1.0
+    return 1.0 - dot / (norm_a * norm_b)
+
+
+@dataclass(frozen=True)
+class SearchResult:
+    artifact_id: str
+    rel_path: str
+    kind: str
+    name: str
+    language: str
+    start_line: int
+    end_line: int
+    code: str
+    distance: float
+    source: str = ""  # ingest source label, "" for pre-18C chunks
+
+
+@dataclass(frozen=True)
+class SyncStats:
+    """Outcome of an incremental sync — every artifact is accounted for."""
+
+    added: int
+    changed: int
+    unchanged: int
+    removed: int
+
+    @property
+    def embedded(self) -> int:
+        return self.added + self.changed
+
+
+class KnowledgeBase:
+    """Vector store over code artifacts backed by ChromaDB."""
+
+    def __init__(self, persist_dir: Path, collection_name: str = "artifacts"):
+        self._persist_dir = Path(persist_dir)
+        self._collection_name = collection_name
+        self._client = chromadb.PersistentClient(path=str(self._persist_dir))
+        self._collection = self._client.get_or_create_collection(
+            name=collection_name,
+            metadata={"hnsw:space": "cosine"},
+        )
+        # Lazy BM25 index over the collection's documents (Phase 22A).
+        # Derived from the same store, invalidated on every write — it can
+        # never drift out of sync with the vectors.
+        self._bm25: BM25Index | None = None
+        self._bm25_source: dict[str, str] = {}
+
+    def _bm25_index(self) -> BM25Index:
+        if self._bm25 is None:
+            got = self._collection.get(include=["documents", "metadatas"])
+            docs: dict[str, str] = {}
+            self._bm25_source = {}
+            for id_, doc, meta in zip(
+                got["ids"],
+                got["documents"] or [],
+                got["metadatas"] or [],
+                strict=False,
+            ):
+                docs[id_] = doc or ""
+                self._bm25_source[id_] = str((meta or {}).get("source", ""))
+            self._bm25 = BM25Index(docs)
+        return self._bm25
+
+    def add_artifacts(
+        self,
+        artifacts: list[Artifact],
+        provider: LLMProvider,
+        *,
+        source: str = "",
+        batch_size: int = _EMBED_BATCH_SIZE,
+        on_progress: Callable[[int, int], None] | None = None,
+    ) -> int:
+        """Embed and store artifacts. Returns the number stored.
+
+        ``on_progress(stored_so_far, total)`` fires after each batch so
+        callers can show progress during slow (rate-limited) embedding runs.
+        """
+        if not artifacts:
+            return 0
+
+        stored = 0
+        for start in range(0, len(artifacts), batch_size):
+            batch = artifacts[start : start + batch_size]
+            texts = [a.code for a in batch]
+            embeddings = provider.embed(texts)
+            ids = [_artifact_id(a) for a in batch]
+            metadatas = [_artifact_meta(a, source) for a in batch]
+            self._collection.upsert(
+                ids=ids,
+                embeddings=embeddings,
+                documents=texts,
+                metadatas=metadatas,
+            )
+            stored += len(batch)
+            if on_progress is not None:
+                on_progress(stored, len(artifacts))
+        self._bm25 = None  # collection changed — rebuild lexical index lazily
+        return stored
+
+    def sync_artifacts(
+        self,
+        artifacts: list[Artifact],
+        provider: LLMProvider,
+        *,
+        source: str = "",
+        batch_size: int = _EMBED_BATCH_SIZE,
+        on_progress: Callable[[int, int], None] | None = None,
+    ) -> SyncStats:
+        """Incremental upsert: embed only new/changed artifacts, drop stale ones.
+
+        Change detection compares the content hash stored in each chunk's
+        metadata, so re-ingesting after a small edit re-embeds only what
+        actually changed — the rest of the corpus is skipped entirely.
+        ``on_progress`` receives (embedded_so_far, total_to_embed), i.e. only
+        the work being done, not the artifacts being skipped.
+        """
+        # Stable identity — duplicates keep the first occurrence.
+        unique: dict[str, Artifact] = {}
+        for a in artifacts:
+            unique.setdefault(_artifact_id(a), a)
+
+        existing: dict[str, str] = {}
+        got = self._collection.get(include=["metadatas"])
+        for id_, meta in zip(got["ids"], got["metadatas"] or [], strict=False):
+            existing[id_] = (meta or {}).get("content_hash", "")
+
+        added = changed = unchanged = 0
+        to_embed: list[Artifact] = []
+        for id_, a in unique.items():
+            if id_ not in existing:
+                added += 1
+                to_embed.append(a)
+            elif existing[id_] != _content_hash(a):
+                changed += 1
+                to_embed.append(a)
+            else:
+                unchanged += 1
+
+        stale = [id_ for id_ in existing if id_ not in unique]
+        if stale:
+            self._collection.delete(ids=stale)
+            self._bm25 = None
+
+        if to_embed:
+            self.add_artifacts(
+                to_embed,
+                provider,
+                source=source,
+                batch_size=batch_size,
+                on_progress=on_progress,
+            )
+
+        return SyncStats(
+            added=added,
+            changed=changed,
+            unchanged=unchanged,
+            removed=len(stale),
+        )
+
+    def search(
+        self,
+        query: str,
+        provider: LLMProvider,
+        n_results: int = 5,
+        *,
+        source: str | None = None,
+        hybrid: bool | None = None,
+    ) -> list[SearchResult]:
+        """Return the closest artifacts for a query.
+
+        Hybrid mode (default, ``RECEIPTS_HYBRID=0`` to disable) runs BOTH
+        retrievers — dense cosine similarity and BM25 lexical matching —
+        and merges their rankings with reciprocal rank fusion. BM25 catches
+        the exact identifiers embeddings blur ("pgvector", "Alembic");
+        crucially, every fused candidate still carries its TRUE semantic
+        distance, so the verifier's evidence threshold gates hybrid results
+        exactly as strictly as dense-only ones. More recall, same honesty.
+
+        ``source`` restricts results to chunks ingested from that source
+        label — the guard against cross-source evidence contamination in
+        mixed corpora (a claim about project A must not "verify" against
+        lookalike code from project B).
+        """
+        count = self._collection.count()
+        if count == 0:
+            return []
+        if hybrid is None:
+            hybrid = _hybrid_enabled()
+
+        n_results = min(n_results, count)
+        embedding = provider.embed([query])[0]
+
+        # Dense leg: over-fetch in hybrid mode so fusion has candidates.
+        fetch_n = min(count, max(n_results * 3, 10)) if hybrid else n_results
+        raw = self._collection.query(
+            query_embeddings=[embedding],
+            n_results=fetch_n,
+            where={"source": source} if source else None,
+            include=["documents", "metadatas", "distances"],
+        )
+        info: dict[str, tuple[dict, str, float]] = {}
+        dense_ids: list[str] = []
+        for i in range(len(raw["ids"][0])):
+            id_ = raw["ids"][0][i]
+            dense_ids.append(id_)
+            info[id_] = (
+                raw["metadatas"][0][i] or {},
+                raw["documents"][0][i],
+                raw["distances"][0][i],
+            )
+
+        if hybrid:
+            index = self._bm25_index()
+            sparse = [id_ for id_, _ in index.search(query, fetch_n)]
+            if source:
+                sparse = [id_ for id_ in sparse if self._bm25_source.get(id_) == source]
+            chosen = rrf_fuse([dense_ids, sparse])[:n_results]
+
+            # BM25-only candidates lack a dense distance — compute the real
+            # cosine distance so the threshold gate treats them identically.
+            missing = [id_ for id_ in chosen if id_ not in info]
+            if missing:
+                got = self._collection.get(
+                    ids=missing,
+                    include=["documents", "metadatas", "embeddings"],
+                )
+                for id_, doc, meta, emb in zip(
+                    got["ids"],
+                    got["documents"] or [],
+                    got["metadatas"] or [],
+                    got["embeddings"],
+                    strict=False,
+                ):
+                    dist = _cosine_distance(embedding, list(emb))
+                    info[id_] = (meta or {}, doc or "", dist)
+        else:
+            chosen = dense_ids[:n_results]
+
+        results: list[SearchResult] = []
+        for id_ in chosen:
+            meta, doc, dist = info[id_]
+            results.append(
+                SearchResult(
+                    artifact_id=id_,
+                    rel_path=meta.get("rel_path", ""),
+                    kind=meta.get("kind", ""),
+                    name=meta.get("name", ""),
+                    language=meta.get("language", ""),
+                    start_line=int(meta.get("start_line", 0)),
+                    end_line=int(meta.get("end_line", 0)),
+                    code=doc,
+                    distance=dist,
+                    source=str(meta.get("source", "")),
+                )
+            )
+        # Fusion decides WHICH results surface; distance orders them, so
+        # callers keep the closest-first contract they always had.
+        results.sort(key=lambda r: r.distance)
+        return results
+
+    def count(self) -> int:
+        return self._collection.count()
+
+    def clear(self) -> None:
+        """Drop all stored artifacts and recreate the collection."""
+        self._client.delete_collection(self._collection_name)
+        self._collection = self._client.get_or_create_collection(
+            name=self._collection_name,
+            metadata={"hnsw:space": "cosine"},
+        )
+        self._bm25 = None
+        self._bm25_source = {}
+
+
+def default_kb_dir(project_root: Path) -> Path:
+    """Derive a per-project KB directory under ``~/.receipts/kb/``."""
+    resolved = str(project_root.resolve())
+    short_hash = hashlib.sha256(resolved.encode()).hexdigest()[:8]
+    name = project_root.resolve().name
+    return Path.home() / ".receipts" / "kb" / f"{name}_{short_hash}"
+
+
+def _artifact_id(artifact: Artifact) -> str:
+    key = f"{artifact.rel_path}:{artifact.kind}:{artifact.name}:{artifact.start_line}"
+    return hashlib.sha256(key.encode()).hexdigest()[:16]
+
+
+def _content_hash(artifact: Artifact) -> str:
+    """Hash of the artifact's code — the change signal for incremental sync."""
+    return hashlib.sha256(artifact.code.encode("utf-8", errors="replace")).hexdigest()[
+        :16
+    ]
+
+
+def _artifact_meta(artifact: Artifact, source: str = "") -> dict[str, str | int]:
+    return {
+        "rel_path": artifact.rel_path,
+        "kind": artifact.kind,
+        "name": artifact.name,
+        "language": artifact.language,
+        "start_line": artifact.start_line,
+        "end_line": artifact.end_line,
+        "content_hash": _content_hash(artifact),
+        "source": source,
+    }

receipts/verify/keyword_gap.py

@@ -0,0 +1,127 @@
+"""JD keyword gap analysis grounded in code evidence.
+
+Given a job description, extract technical keywords and report which
+ones the resume already covers, which the codebase can justify adding,
+and which have no support in either place.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+
+from receipts.llm.json_utils import parse_json_loose
+from receipts.llm.provider import CompletionResult, LLMProvider
+from receipts.resume.tex_parser import ParsedResume
+from receipts.verify.kb import KnowledgeBase
+
+_EXTRACT_SYSTEM = """\
+Extract the key technical skills, tools, frameworks, and qualifications \
+from this job description. Return ONLY concrete, specific skills \
+(e.g. "React", "PostgreSQL", "CI/CD", "REST APIs") — no soft skills \
+or generic phrases.
+
+Respond with a JSON array: ["skill1", "skill2", ...]
+"""
+
+
+@dataclass(frozen=True)
+class KeywordGapItem:
+    keyword: str
+    in_resume: bool
+    in_codebase: bool
+    suggestion: str
+
+
+@dataclass(frozen=True)
+class KeywordGapReport:
+    items: list[KeywordGapItem]
+    completion: CompletionResult | None = None
+
+    @property
+    def addable(self) -> list[KeywordGapItem]:
+        """Keywords the code supports but resume doesn't mention."""
+        return [i for i in self.items if not i.in_resume and i.in_codebase]
+
+    @property
+    def ungrounded(self) -> list[KeywordGapItem]:
+        """Keywords the resume claims but code doesn't back up."""
+        return [i for i in self.items if i.in_resume and not i.in_codebase]
+
+    @property
+    def gaps(self) -> list[KeywordGapItem]:
+        """Keywords in the JD absent from both resume and codebase."""
+        return [i for i in self.items if not i.in_resume and not i.in_codebase]
+
+
+def _extract_jd_keywords(
+    jd_text: str, provider: LLMProvider
+) -> tuple[list[str], CompletionResult]:
+    completion = provider.complete(_EXTRACT_SYSTEM, jd_text, json_mode=True)
+    try:
+        keywords = parse_json_loose(completion.text)
+        if isinstance(keywords, list):
+            return [str(k) for k in keywords], completion
+        # Some models wrap the array: {"skills": [...]} or {"keywords": [...]}
+        if isinstance(keywords, dict):
+            for value in keywords.values():
+                if isinstance(value, list):
+                    return [str(k) for k in value], completion
+    except (ValueError, json.JSONDecodeError, AttributeError):
+        pass
+    return [], completion
+
+
+def _resume_text(resume: ParsedResume) -> str:
+    """Flatten the parsed resume into a single searchable string."""
+    parts: list[str] = []
+    for sec in resume.sections:
+        parts.append(sec.name)
+        for entry in sec.entries:
+            parts.append(entry.heading)
+            for b in entry.bullets:
+                parts.append(b.text)
+    return " ".join(parts).lower()
+
+
+def analyze_gap(
+    jd_text: str,
+    resume: ParsedResume,
+    kb: KnowledgeBase,
+    provider: LLMProvider,
+    *,
+    distance_threshold: float = 0.5,
+) -> KeywordGapReport:
+    """Analyse the gap between a JD, resume, and code evidence."""
+    keywords, completion = _extract_jd_keywords(jd_text, provider)
+    if not keywords:
+        return KeywordGapReport(items=[], completion=completion)
+
+    flat = _resume_text(resume)
+
+    items: list[KeywordGapItem] = []
+    for kw in keywords:
+        in_resume = kw.lower() in flat
+
+        results = kb.search(kw, provider, n_results=3)
+        in_codebase = any(r.distance <= distance_threshold for r in results)
+
+        if not in_resume and in_codebase:
+            suggestion = f"Your code uses {kw} — consider adding it to your resume."
+        elif in_resume and not in_codebase:
+            suggestion = f"Resume mentions {kw} but no code evidence found."
+        elif not in_resume and not in_codebase:
+            suggestion = f"JD requires {kw} — not in resume or codebase."
+        else:
+            suggestion = f"{kw} is in your resume and backed by code."
+
+        items.append(
+            KeywordGapItem(
+                keyword=kw,
+                in_resume=in_resume,
+                in_codebase=in_codebase,
+                suggestion=suggestion,
+            )
+        )
+
+    return KeywordGapReport(items=items, completion=completion)

receipts/verify/query_expand.py

@@ -0,0 +1,88 @@
+"""Query-side intelligence (Phase 22D) — search the way code is written.
+
+Resume claims are phrased for recruiters ("reduced query latency by 40%");
+code is phrased for machines ("CREATE INDEX", "cache.get", "asyncio.gather").
+Two techniques bridge the gap, both using the configured provider, both
+metered, both OFF by default (``RECEIPTS_QUERY_EXPANSION=1`` to enable):
+
+- **Multi-query expansion**: rewrite the claim into 2-3 short queries
+  phrased the way the proving code would look.
+- **HyDE** (hypothetical document embeddings): generate a small snippet of
+  code that WOULD prove the claim, and search with that — a fake document
+  finds real documents better than an abstract claim does.
+
+Plus the ONE bounded reformulate-and-retry hop (Phase 22E's retry half):
+if the first retrieval finds nothing above threshold, reformulate once and
+try again. Once. Never an unbounded agent loop.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+
+from receipts.llm.json_utils import parse_json_loose
+from receipts.llm.provider import LLMProvider
+
+_EXPAND_SYSTEM = """\
+You turn resume claims into code-search queries. Given a claim, write 2-3
+SHORT queries (3-8 words each) phrased the way the implementing code would
+look — function names, library names, config keys, SQL keywords.
+
+Respond in JSON: {"queries": ["q1", "q2", "q3"]}
+"""
+
+_HYDE_SYSTEM = """\
+You write hypothetical evidence. Given a resume claim, write a SHORT code
+snippet (max 12 lines, any language that fits) that WOULD exist in a
+codebase where the claim is true. Realistic identifiers, no comments
+explaining yourself.
+
+Respond in JSON: {"code": "the snippet"}
+"""
+
+_REFORMULATE_SYSTEM = """\
+A code search for the claim below found nothing relevant. Write ONE
+alternative search query (3-10 words) using different, more concrete
+vocabulary — library names, patterns, or synonyms the code might use.
+
+Respond in JSON: {"query": "the query"}
+"""
+
+
+def expansion_enabled() -> bool:
+    """22D toggle — off unless RECEIPTS_QUERY_EXPANSION=1."""
+    return os.environ.get("RECEIPTS_QUERY_EXPANSION", "0").strip() == "1"
+
+
+def expand_queries(claim_text: str, provider: LLMProvider) -> list[str]:
+    """2-3 code-phrased search queries for the claim (may be empty)."""
+    try:
+        completion = provider.complete(_EXPAND_SYSTEM, claim_text, json_mode=True)
+        parsed = parse_json_loose(completion.text)
+        raw = parsed.get("queries", [])
+        if isinstance(raw, list):
+            return [str(q).strip() for q in raw if str(q).strip()][:3]
+    except (ValueError, json.JSONDecodeError, AttributeError, Exception):
+        pass
+    return []
+
+
+def hyde_snippet(claim_text: str, provider: LLMProvider) -> str:
+    """A hypothetical code snippet that would prove the claim ('' on failure)."""
+    try:
+        completion = provider.complete(_HYDE_SYSTEM, claim_text, json_mode=True)
+        parsed = parse_json_loose(completion.text)
+        return str(parsed.get("code", "")).strip()[:1200]
+    except (ValueError, json.JSONDecodeError, AttributeError, Exception):
+        return ""
+
+
+def reformulate(claim_text: str, provider: LLMProvider) -> str:
+    """One alternative query for the single bounded retry hop ('' on failure)."""
+    try:
+        completion = provider.complete(_REFORMULATE_SYSTEM, claim_text, json_mode=True)
+        parsed = parse_json_loose(completion.text)
+        return str(parsed.get("query", "")).strip()[:200]
+    except (ValueError, json.JSONDecodeError, AttributeError, Exception):
+        return ""

receipts/verify/rerank.py

@@ -0,0 +1,74 @@
+"""LLM-as-reranker (Phase 22E) — sharpen the top of the retrieval pile.
+
+Similarity ranking is approximate: the truly relevant chunk is usually in
+the top 20 but not always in the top 5. The reranker shows the model the
+claim plus a numbered candidate list (paths, names, first lines) and asks
+which candidates actually bear on the claim, in order.
+
+Honesty properties: the reranker can only REORDER and DROP existing
+candidates — it cannot add anything, and every candidate keeps its true
+semantic distance, so the verifier's threshold still applies afterwards.
+On any parse failure the original ranking is kept. OFF by default
+(``RECEIPTS_RERANK=1`` to enable); works on Ollama/Gemini — no
+cross-encoder dependency.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+
+from receipts.llm.json_utils import parse_json_loose
+from receipts.llm.provider import LLMProvider
+from receipts.verify.kb import SearchResult
+
+_RERANK_SYSTEM = """\
+You rank code-search candidates by relevance to a claim. You will get the
+claim and a numbered list of code chunks. Reply with the numbers of the
+chunks MOST relevant to judging the claim, best first. Only include
+genuinely relevant ones — fewer is fine.
+
+Respond in JSON: {"keep": [3, 1, 7]}
+"""
+
+
+def rerank_enabled() -> bool:
+    """22E toggle — off unless RECEIPTS_RERANK=1."""
+    return os.environ.get("RECEIPTS_RERANK", "0").strip() == "1"
+
+
+def rerank(
+    claim_text: str,
+    candidates: list[SearchResult],
+    provider: LLMProvider,
+    *,
+    keep: int = 5,
+) -> list[SearchResult]:
+    """Rerank candidates for the claim; fall back to input order on failure."""
+    if len(candidates) <= keep:
+        return candidates
+
+    lines = [f"CLAIM: {claim_text}", "", "CANDIDATES:"]
+    for i, r in enumerate(candidates, 1):
+        head = " ".join(r.code[:220].split())
+        lines.append(f"[{i}] {r.rel_path} :: {r.name} ({r.kind}) — {head}")
+
+    try:
+        completion = provider.complete(_RERANK_SYSTEM, "\n".join(lines), json_mode=True)
+        parsed = parse_json_loose(completion.text)
+        raw = parsed.get("keep", [])
+        picked: list[SearchResult] = []
+        seen: set[int] = set()
+        for idx in raw if isinstance(raw, list) else []:
+            try:
+                pos = int(idx)
+            except (TypeError, ValueError):
+                continue
+            if 1 <= pos <= len(candidates) and pos not in seen:
+                seen.add(pos)
+                picked.append(candidates[pos - 1])
+        if picked:
+            return picked[:keep]
+    except (ValueError, json.JSONDecodeError, AttributeError, Exception):
+        pass
+    return candidates[:keep]