codebase-receipts-cli 1.0.0__py3-none-any.whl

receipts/verify/verifier.py

@@ -0,0 +1,552 @@
+"""Verify resume claims against the code knowledge base.
+
+For each claim, the verifier searches the KB for evidence and classifies
+it as Verified, Plausible, or Unsupported.  When no evidence is found
+above the similarity threshold the claim is classified Unsupported
+*directly* — the LLM is never asked to rationalise a claim it cannot
+find support for.
+
+Section-aware behavior (Phase 8):
+- **project** sections: full KB search + LLM verification (original behavior).
+- **experience** sections: same as project when a KB exists for that role's
+  codebase; otherwise classified as ``plausible`` with a note.
+- **education / certifications / publications**: always ``plausible`` — these
+  are unprovable from code.
+- **skills**: cross-referenced against the KB — if the skill appears in code
+  artifacts, ``verified``; if related code exists, ``plausible``; otherwise
+  ``unsupported``.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+
+from receipts.llm.json_utils import parse_json_loose
+from receipts.llm.provider import CompletionResult, LLMProvider
+from receipts.resume.claim_extractor import Claim
+from receipts.verify.kb import KnowledgeBase, SearchResult
+
+DISTANCE_THRESHOLD = 0.5
+
+_SYSTEM = """\
+You are a resume claim verifier. You will receive a resume claim and \
+code evidence retrieved from the project's codebase. Decide whether \
+the evidence supports the claim.
+
+Verdicts
+--------
+* "verified"   — the evidence directly and clearly proves the claim.
+* "plausible"  — the evidence is related but does not fully confirm it.
+* "unsupported"— the evidence does not support the claim.
+
+Rules
+-----
+- Base your verdict ONLY on the provided evidence. Never infer or guess.
+- If evidence is thin or ambiguous, prefer "plausible" over "verified".
+- For numeric claims the code must show the numbers or the mechanism.
+- For technology claims the code must import, configure, or use it.
+
+Respond in JSON:
+{"verdict": "verified|plausible|unsupported", "explanation": "one sentence", \
+"cited_ids": ["id1"]}
+"""
+
+_SKILLS_SYSTEM = """\
+You are a resume skills verifier. You will receive a skill claim and \
+code evidence retrieved from the project's codebase. Decide whether \
+the codebase demonstrates use of this skill/technology.
+
+Verdicts
+--------
+* "verified"   — the code clearly imports, configures, or uses this technology.
+* "plausible"  — the code is related but doesn't directly demonstrate the skill.
+* "unsupported"— no evidence of this skill in the codebase.
+
+Respond in JSON:
+{"verdict": "verified|plausible|unsupported", "explanation": "one sentence", \
+"cited_ids": ["id1"]}
+"""
+
+_BATCH_SYSTEM = """\
+You are a resume claim verifier. You will receive SEVERAL claims that all
+come from the same resume bullet, plus code evidence retrieved from the
+project's codebase. Decide, for EACH claim independently, whether the
+evidence supports it.
+
+Verdicts
+--------
+* "verified"   — the evidence directly and clearly proves the claim.
+* "plausible"  — the evidence is related but does not fully confirm it.
+* "unsupported"— the evidence does not support the claim.
+
+Rules
+-----
+- Base every verdict ONLY on the provided evidence. Never infer or guess.
+- If evidence is thin or ambiguous, prefer "plausible" over "verified".
+- For numeric claims the code must show the numbers or the mechanism.
+- For technology claims the code must import, configure, or use it.
+- Judge each claim on its own — one verified claim does not make its
+  neighbors verified.
+
+Respond in JSON:
+{"verdicts": [{"index": 1, "verdict": "verified|plausible|unsupported", \
+"explanation": "one sentence"}, ...]}
+Include one entry per claim, using the claim's index.
+"""
+
+_BATCH_SKILLS_SYSTEM = """\
+You are a resume skills verifier. You will receive SEVERAL skill claims,
+plus code evidence retrieved from the project's codebase. Decide, for
+EACH skill independently, whether the codebase demonstrates it.
+
+Verdicts
+--------
+* "verified"   — the code clearly imports, configures, or uses this technology.
+* "plausible"  — the code is related but doesn't directly demonstrate the skill.
+* "unsupported"— no evidence of this skill in the codebase.
+
+Respond in JSON:
+{"verdicts": [{"index": 1, "verdict": "verified|plausible|unsupported", \
+"explanation": "one sentence"}, ...]}
+Include one entry per claim, using the claim's index.
+"""
+
+_NOPROBE_SECTIONS = {"education", "other"}
+
+
+@dataclass(frozen=True)
+class VerificationResult:
+    claim: Claim
+    verdict: str  # "verified" | "plausible" | "unsupported"
+    explanation: str
+    evidence: list[SearchResult] = field(default_factory=list)
+    llm_used: bool = False
+    completion: CompletionResult | None = None
+
+
+def _build_user_prompt(claim: Claim, evidence: list[SearchResult]) -> str:
+    parts = [
+        f"CLAIM ({claim.claim_type}): {claim.value}",
+        f"CONTEXT: {claim.snippet}",
+        f"BULLET: {claim.bullet_text}",
+        "",
+        "CODE EVIDENCE:",
+    ]
+    for e in evidence:
+        parts.append(f"--- [{e.artifact_id}] {e.rel_path} :: {e.name} ({e.kind}) ---")
+        parts.append(e.code[:2000])
+        parts.append("")
+    return "\n".join(parts)
+
+
+def _merge_pools(pools: list[list[SearchResult]]) -> list[SearchResult]:
+    """Union result pools, dedupe by artifact, keep each one's best distance."""
+    best: dict[str, SearchResult] = {}
+    for pool in pools:
+        for r in pool:
+            cur = best.get(r.artifact_id)
+            if cur is None or r.distance < cur.distance:
+                best[r.artifact_id] = r
+    return sorted(best.values(), key=lambda r: r.distance)
+
+
+def _retrieve_evidence(
+    query: str,
+    kb: KnowledgeBase,
+    provider: LLMProvider,
+    *,
+    n_results: int,
+    distance_threshold: float,
+) -> list[SearchResult]:
+    """Retrieve threshold-passing evidence for a query.
+
+    The base path is one KB search. Two opt-in upgrades layer on top:
+
+    - 22D (``RECEIPTS_QUERY_EXPANSION=1``): also search 2-3 code-phrased
+      rewrites of the claim plus a HyDE snippet, merge the pools, and — if
+      nothing passes the threshold — reformulate ONCE and retry. Bounded.
+    - 22E (``RECEIPTS_RERANK=1``): over-fetch, let the model reorder the
+      top candidates, keep the best few.
+
+    Every path ends at the same gate: only results whose true semantic
+    distance clears the threshold count as evidence. The upgrades widen
+    the net; they never loosen the mesh.
+    """
+    from receipts.verify.query_expand import (
+        expand_queries,
+        expansion_enabled,
+        hyde_snippet,
+        reformulate,
+    )
+    from receipts.verify.rerank import rerank as _rerank
+    from receipts.verify.rerank import rerank_enabled
+
+    use_expand = expansion_enabled()
+    use_rerank = rerank_enabled()
+    fetch_n = max(n_results * 4, 20) if use_rerank else n_results
+
+    pools = [kb.search(query, provider, n_results=fetch_n)]
+    if use_expand:
+        for q in expand_queries(query, provider):
+            pools.append(kb.search(q, provider, n_results=fetch_n))
+        snippet = hyde_snippet(query, provider)
+        if snippet:
+            pools.append(kb.search(snippet, provider, n_results=fetch_n))
+    merged = _merge_pools(pools)
+
+    if use_rerank and len(merged) > n_results:
+        merged = _rerank(query, merged[:20], provider, keep=n_results)
+    else:
+        merged = merged[:n_results]
+
+    relevant = [r for r in merged if r.distance <= distance_threshold]
+
+    if not relevant and use_expand:
+        # ONE bounded reformulate-and-retry hop — never a loop.
+        retry_q = reformulate(query, provider)
+        if retry_q and retry_q.lower() != query.lower():
+            retry_pool = kb.search(retry_q, provider, n_results=n_results)
+            relevant = [r for r in retry_pool if r.distance <= distance_threshold]
+    return relevant
+
+
+def verify_claim(
+    claim: Claim,
+    kb: KnowledgeBase,
+    provider: LLMProvider,
+    *,
+    distance_threshold: float = DISTANCE_THRESHOLD,
+    n_results: int = 5,
+) -> VerificationResult:
+    """Verify a single claim against the knowledge base.
+
+    Section-aware: education/certification/publication claims are always
+    classified as ``plausible`` (no code evidence possible). Skills claims
+    are cross-referenced against the KB. Experience/project claims get
+    full verification.
+    """
+    sec_type = getattr(claim, "section_type", "project")
+
+    if sec_type in _NOPROBE_SECTIONS and claim.claim_type == "contextual":
+        return VerificationResult(
+            claim=claim,
+            verdict="plausible",
+            explanation=f"No code verification possible for {claim.section} claims.",
+            evidence=[],
+            llm_used=False,
+        )
+
+    query = f"{claim.value} {claim.snippet}"
+    relevant = _retrieve_evidence(
+        query,
+        kb,
+        provider,
+        n_results=n_results,
+        distance_threshold=distance_threshold,
+    )
+
+    if not relevant:
+        if sec_type == "experience" and claim.claim_type == "contextual":
+            return VerificationResult(
+                claim=claim,
+                verdict="plausible",
+                explanation=(
+                    "No codebase provided for this role — claim is"
+                    " plausible but unverifiable."
+                ),
+                evidence=[],
+                llm_used=False,
+            )
+        return VerificationResult(
+            claim=claim,
+            verdict="unsupported",
+            explanation="No relevant code evidence found in the knowledge base.",
+            evidence=[],
+            llm_used=False,
+        )
+
+    system_prompt = _SKILLS_SYSTEM if sec_type == "skills" else _SYSTEM
+    user_prompt = _build_user_prompt(claim, relevant)
+    completion = provider.complete(system_prompt, user_prompt, json_mode=True)
+
+    try:
+        parsed = parse_json_loose(completion.text)
+        verdict = str(parsed.get("verdict", "unsupported")).lower()
+        if verdict not in ("verified", "plausible", "unsupported"):
+            verdict = "unsupported"
+        explanation = parsed.get("explanation", "")
+    except (ValueError, json.JSONDecodeError, AttributeError):
+        verdict = "plausible"
+        explanation = completion.text[:200]
+
+    return VerificationResult(
+        claim=claim,
+        verdict=verdict,
+        explanation=explanation,
+        evidence=relevant,
+        llm_used=True,
+        completion=completion,
+    )
+
+
+def _short_circuit(claim: Claim) -> VerificationResult | None:
+    """Return a result for claims that never need the KB or the LLM."""
+    sec_type = getattr(claim, "section_type", "project")
+    if sec_type in _NOPROBE_SECTIONS and claim.claim_type == "contextual":
+        return VerificationResult(
+            claim=claim,
+            verdict="plausible",
+            explanation=f"No code verification possible for {claim.section} claims.",
+            evidence=[],
+            llm_used=False,
+        )
+    return None
+
+
+def _no_evidence_result(claim: Claim) -> VerificationResult:
+    sec_type = getattr(claim, "section_type", "project")
+    if sec_type == "experience" and claim.claim_type == "contextual":
+        return VerificationResult(
+            claim=claim,
+            verdict="plausible",
+            explanation=(
+                "No codebase provided for this role — claim is"
+                " plausible but unverifiable."
+            ),
+            evidence=[],
+            llm_used=False,
+        )
+    return VerificationResult(
+        claim=claim,
+        verdict="unsupported",
+        explanation="No relevant code evidence found in the knowledge base.",
+        evidence=[],
+        llm_used=False,
+    )
+
+
+_VALID_VERDICTS = ("verified", "plausible", "unsupported")
+
+
+def _parse_batch_verdicts(
+    text: str, probe_claims: list[Claim]
+) -> dict[int, tuple[str, str]]:
+    """Normalize the many shapes small models return for batch verdicts.
+
+    Accepted shapes (all observed in the wild from lite-tier models):
+    - {"verdicts": [{"index": 1, "verdict": ..., "explanation": ...}, ...]}
+    - {"verdicts": {"1": {"verdict": ...}}} or {"verdicts": {"1": "verified"}}
+    - a bare top-level list of entry objects
+    - a single {"verdict": ..., "explanation": ...} object (one claim)
+    - entries with a string index ("1"), no index (positional), or a
+      "claim" key naming the claim value instead of an index
+    Anything unrecognizable is simply absent from the result — the caller's
+    honest per-claim fallback handles it. Never guess a verdict.
+    """
+    try:
+        parsed = parse_json_loose(text)
+    except (ValueError, json.JSONDecodeError):
+        return {}
+
+    value_to_index = {
+        c.value.strip().lower(): i + 1 for i, c in enumerate(probe_claims)
+    }
+
+    def _clean_verdict(raw) -> str | None:
+        v = str(raw or "").strip().lower()
+        if not v:
+            return None
+        return v if v in _VALID_VERDICTS else "unsupported"
+
+    def _entry_index(entry: dict, position: int) -> int | None:
+        idx = entry.get("index")
+        if idx is not None:
+            try:
+                return int(idx)
+            except (TypeError, ValueError):
+                pass
+        claim_key = str(entry.get("claim", "")).strip().lower()
+        if claim_key and claim_key in value_to_index:
+            return value_to_index[claim_key]
+        return position  # fall back to positional order (1-based)
+
+    out: dict[int, tuple[str, str]] = {}
+
+    if isinstance(parsed, dict) and "verdicts" in parsed:
+        verdicts = parsed["verdicts"]
+    elif isinstance(parsed, list):
+        verdicts = parsed
+    elif isinstance(parsed, dict) and "verdict" in parsed:
+        verdict = _clean_verdict(parsed.get("verdict"))
+        if verdict:
+            out[1] = (verdict, str(parsed.get("explanation", "")))
+        return out
+    else:
+        return out
+
+    if isinstance(verdicts, dict):
+        # {"1": {...}} or {"1": "verified"}
+        for key, value in verdicts.items():
+            try:
+                idx = int(key)
+            except (TypeError, ValueError):
+                idx = value_to_index.get(str(key).strip().lower(), 0)
+            if idx <= 0:
+                continue
+            if isinstance(value, dict):
+                verdict = _clean_verdict(value.get("verdict"))
+                explanation = str(value.get("explanation", ""))
+            else:
+                verdict = _clean_verdict(value)
+                explanation = ""
+            if verdict:
+                out[idx] = (verdict, explanation)
+        return out
+
+    if isinstance(verdicts, list):
+        for position, entry in enumerate(verdicts, 1):
+            if not isinstance(entry, dict):
+                continue
+            verdict = _clean_verdict(entry.get("verdict"))
+            if verdict is None:
+                continue
+            idx = _entry_index(entry, position)
+            if idx is not None and idx > 0:
+                out[idx] = (verdict, str(entry.get("explanation", "")))
+
+    return out
+
+
+def _build_batch_prompt(claims: list[Claim], evidence: list[SearchResult]) -> str:
+    parts = ["CLAIMS (all from the same resume bullet):"]
+    for i, c in enumerate(claims, 1):
+        parts.append(f"{i}. ({c.claim_type}) {c.value} — context: {c.snippet}")
+    parts.append("")
+    parts.append(f"BULLET: {claims[0].bullet_text}")
+    parts.append("")
+    parts.append("CODE EVIDENCE:")
+    for e in evidence:
+        parts.append(f"--- [{e.artifact_id}] {e.rel_path} :: {e.name} ({e.kind}) ---")
+        parts.append(e.code[:2000])
+        parts.append("")
+    return "\n".join(parts)
+
+
+def verify_bullet_claims(
+    claims: list[Claim],
+    kb: KnowledgeBase,
+    provider: LLMProvider,
+    *,
+    distance_threshold: float = DISTANCE_THRESHOLD,
+    n_results: int = 5,
+) -> list[VerificationResult]:
+    """Verify all claims from ONE bullet with a single KB search + LLM call.
+
+    Claims sharing a bullet share the same code context, so retrieving
+    evidence once and asking the model to judge each claim independently
+    in one response cuts API calls roughly 3x without softening any
+    verdict — the no-evidence guard still short-circuits to Unsupported
+    before the LLM is ever consulted.
+    """
+    if not claims:
+        return []
+
+    results: list[VerificationResult | None] = [None] * len(claims)
+    probe_indices: list[int] = []
+    for i, claim in enumerate(claims):
+        short = _short_circuit(claim)
+        if short is not None:
+            results[i] = short
+        else:
+            probe_indices.append(i)
+
+    if not probe_indices:
+        return [r for r in results if r is not None]
+
+    probe_claims = [claims[i] for i in probe_indices]
+
+    # One KB search covering the bullet and every claim value in it.
+    values = " ".join(c.value for c in probe_claims)
+    query = f"{probe_claims[0].bullet_text} {values}"
+    relevant = _retrieve_evidence(
+        query,
+        kb,
+        provider,
+        n_results=n_results,
+        distance_threshold=distance_threshold,
+    )
+
+    if not relevant:
+        for i in probe_indices:
+            results[i] = _no_evidence_result(claims[i])
+        return [r for r in results if r is not None]
+
+    sec_type = getattr(probe_claims[0], "section_type", "project")
+    system_prompt = _BATCH_SKILLS_SYSTEM if sec_type == "skills" else _BATCH_SYSTEM
+    user_prompt = _build_batch_prompt(probe_claims, relevant)
+    completion = provider.complete(system_prompt, user_prompt, json_mode=True)
+
+    verdict_by_index = _parse_batch_verdicts(completion.text, probe_claims)
+
+    for pos, i in enumerate(probe_indices):
+        verdict, explanation = verdict_by_index.get(
+            pos + 1,
+            ("plausible", "Model response could not be parsed for this claim."),
+        )
+        results[i] = VerificationResult(
+            claim=claims[i],
+            verdict=verdict,
+            explanation=explanation,
+            evidence=relevant,
+            llm_used=True,
+            # Attach the shared completion to the first probed claim only,
+            # so token counts aren't double-counted by callers.
+            completion=completion if pos == 0 else None,
+        )
+
+    return [r for r in results if r is not None]
+
+
+def group_claims_by_bullet(claims: list[Claim]) -> list[list[Claim]]:
+    """Group claims that came from the same bullet, preserving order."""
+    groups: list[list[Claim]] = []
+    key_to_group: dict[tuple[str, str, str], list[Claim]] = {}
+    for claim in claims:
+        key = (claim.section, claim.entry_heading, claim.bullet_text)
+        group = key_to_group.get(key)
+        if group is None:
+            group = []
+            key_to_group[key] = group
+            groups.append(group)
+        group.append(claim)
+    return groups
+
+
+def verify_claims(
+    claims: list[Claim],
+    kb: KnowledgeBase,
+    provider: LLMProvider,
+    *,
+    distance_threshold: float = DISTANCE_THRESHOLD,
+    batch: bool = False,
+) -> list[VerificationResult]:
+    """Verify all claims against the knowledge base.
+
+    With ``batch=True``, claims from the same bullet are verified in one
+    KB search + one LLM call — use for rate-limited API providers. Local
+    Ollama models are better served by the simpler per-claim prompt.
+    """
+    if batch:
+        out: list[VerificationResult] = []
+        for group in group_claims_by_bullet(claims):
+            out.extend(
+                verify_bullet_claims(
+                    group, kb, provider, distance_threshold=distance_threshold
+                )
+            )
+        return out
+    return [
+        verify_claim(claim, kb, provider, distance_threshold=distance_threshold)
+        for claim in claims
+    ]