contexttrace 0.1.0__tar.gz → 0.2.0__tar.gz

{contexttrace-0.1.0 → contexttrace-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: contexttrace
-Version: 0.1.0
+Version: 0.2.0
 Summary: Local-first SDK and CLI for RAG and agent reliability tracing, citation checks, and failure diagnosis.
 Author: ContextTrace contributors
 License: MIT
@@ -133,6 +133,30 @@ contexttrace eval \
   --fail-on "failure_rate>0.25"
 ```
 
+## Claim-Level Evidence Verification
+
+Verify a portable RAG trace artifact without a hosted dashboard:
+
+```bash
+contexttrace verify-demo unsupported_claim --report
+contexttrace verify trace.json
+contexttrace verify trace.json --json
+contexttrace verify trace.json --report --out reports/example.html
+contexttrace verify trace.json --mode semantic
+contexttrace verify trace.json --fail-on unsupported --fail-on citation_mismatch
+contexttrace verify-benchmark --mode semantic
+```
+
+Input requires `query`, `answer`, and `contexts` with `id` and `text`. Optional `citations` are checked to catch cited sources that do not actually support the matched claim.
+
+`verify-demo` uses bundled demo traces, so it works immediately after `pip install contexttrace`. Available demos include `unsupported_claim`, `partial_support`, `citation_mismatch`, `should_abstain`, and `supported_answer`.
+
+Use `--mode semantic` for local paraphrase-aware matching, and `verify-benchmark` to inspect bundled precision/recall metrics.
+
+ContextTrace verifies whether each generated claim is actually supported by retrieved evidence. Instead of only showing a trace or a score, it tells you where the evidence chain broke: unsupported claim, citation mismatch, insufficient context, or should-have-abstained.
+
+The v0.2.0 verifier uses local lexical heuristics by default. Claim extraction is rule-based, contradiction detection is conservative, and semantic or LLM-judge support can be added later.
+
 ## What It Catches
 
 - `retrieval_miss`

{contexttrace-0.1.0 → contexttrace-0.2.0}/README.md

@@ -76,6 +76,30 @@ contexttrace eval \
   --fail-on "failure_rate>0.25"
 ```
 
+## Claim-Level Evidence Verification
+
+Verify a portable RAG trace artifact without a hosted dashboard:
+
+```bash
+contexttrace verify-demo unsupported_claim --report
+contexttrace verify trace.json
+contexttrace verify trace.json --json
+contexttrace verify trace.json --report --out reports/example.html
+contexttrace verify trace.json --mode semantic
+contexttrace verify trace.json --fail-on unsupported --fail-on citation_mismatch
+contexttrace verify-benchmark --mode semantic
+```
+
+Input requires `query`, `answer`, and `contexts` with `id` and `text`. Optional `citations` are checked to catch cited sources that do not actually support the matched claim.
+
+`verify-demo` uses bundled demo traces, so it works immediately after `pip install contexttrace`. Available demos include `unsupported_claim`, `partial_support`, `citation_mismatch`, `should_abstain`, and `supported_answer`.
+
+Use `--mode semantic` for local paraphrase-aware matching, and `verify-benchmark` to inspect bundled precision/recall metrics.
+
+ContextTrace verifies whether each generated claim is actually supported by retrieved evidence. Instead of only showing a trace or a score, it tells you where the evidence chain broke: unsupported claim, citation mismatch, insufficient context, or should-have-abstained.
+
+The v0.2.0 verifier uses local lexical heuristics by default. Claim extraction is rule-based, contradiction detection is conservative, and semantic or LLM-judge support can be added later.
+
 ## What It Catches
 
 - `retrieval_miss`

contexttrace-0.2.0/contexttrace/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0"

{contexttrace-0.1.0 → contexttrace-0.2.0}/contexttrace/cli.py

@@ -22,6 +22,15 @@ from contexttrace.regression import BENCHMARK_STRATEGIES, run_local_benchmark
 from contexttrace.report import ReportGenerator
 from contexttrace.storage import SQLiteTraceStore
 from contexttrace.thresholds import parse_thresholds, threshold_failures
+from contexttrace.verify import (
+    VerificationInputError,
+    list_verify_demos,
+    load_trace_file,
+    load_verify_demo,
+    verify_trace,
+)
+from contexttrace.verify.benchmark import run_verify_benchmark
+from contexttrace.verify.report import VerifyReportGenerator
 from contexttrace.viewer import serve_viewer
 
 
@@ -198,6 +207,214 @@ def report(
         webbrowser.open(Path(written).resolve().as_uri())
 
 
+@cli.command("verify")
+@click.argument("trace_json")
+@click.option("--json", "json_output", is_flag=True, help="Print the full verification result as JSON.")
+@click.option("--report", is_flag=True, help="Generate a local HTML verification report.")
+@click.option("--out", default=None, help="HTML report path. Implies --report when provided.")
+@click.option("--mode", default="lexical", show_default=True, type=click.Choice(["lexical", "semantic"]), help="Evidence scoring mode.")
+@click.option("--fail-on", multiple=True, help="Fail on unsupported, partial_support, citation_mismatch, should_abstain, contradicted, unverifiable, no_citation, or any_failure.")
+def verify_command(
+    trace_json: str,
+    json_output: bool,
+    report: bool,
+    out: Optional[str],
+    mode: str,
+    fail_on: tuple[str, ...],
+) -> int:
+    """Verify claim-level evidence support for a portable RAG trace JSON file."""
+
+    try:
+        trace = load_trace_file(trace_json)
+    except VerificationInputError as exc:
+        raise click.ClickException(str(exc)) from exc
+
+    result = verify_trace(trace, mode=mode)
+    written_report = _write_verify_report(
+        result,
+        trace,
+        report=report,
+        out=out,
+        default_name="%s_verify.html" % Path(trace_json).stem,
+    )
+    return _print_verify_result(
+        result,
+        json_output=json_output,
+        written_report=written_report,
+        fail_on=fail_on,
+    )
+
+
+@cli.command("verify-demo")
+@click.argument("demo_name", required=False, default="unsupported_claim")
+@click.option("--json", "json_output", is_flag=True, help="Print the full verification result as JSON.")
+@click.option("--report", is_flag=True, help="Generate a local HTML verification report.")
+@click.option("--out", default=None, help="HTML report path. Implies --report when provided.")
+@click.option("--mode", default="lexical", show_default=True, type=click.Choice(["lexical", "semantic"]), help="Evidence scoring mode.")
+@click.option("--fail-on", multiple=True, help="Fail on unsupported, partial_support, citation_mismatch, should_abstain, contradicted, unverifiable, no_citation, or any_failure.")
+def verify_demo_command(
+    demo_name: str,
+    json_output: bool,
+    report: bool,
+    out: Optional[str],
+    mode: str,
+    fail_on: tuple[str, ...],
+) -> int:
+    """Run a bundled claim-level verification demo."""
+
+    try:
+        trace = load_verify_demo(demo_name)
+    except KeyError as exc:
+        raise click.ClickException(
+            "Unknown verify demo %s. Available demos: %s"
+            % (demo_name, ", ".join(list_verify_demos()))
+        ) from exc
+
+    result = verify_trace(trace, mode=mode)
+    written_report = _write_verify_report(
+        result,
+        trace,
+        report=report,
+        out=out,
+        default_name="%s_verify_demo.html" % demo_name,
+    )
+    return _print_verify_result(
+        result,
+        json_output=json_output,
+        written_report=written_report,
+        fail_on=fail_on,
+    )
+
+
+@cli.command("verify-benchmark")
+@click.option("--mode", default="lexical", show_default=True, type=click.Choice(["lexical", "semantic"]), help="Evidence scoring mode.")
+@click.option("--json", "json_output", is_flag=True, help="Print benchmark results as JSON.")
+def verify_benchmark_command(mode: str, json_output: bool) -> int:
+    """Run the bundled verification precision/recall benchmark."""
+
+    result = run_verify_benchmark(mode=mode)
+    if json_output:
+        click.echo(json.dumps(result, indent=2))
+        return 0
+
+    click.echo("Mode: %s" % result["mode"])
+    click.echo("Cases: %s" % result["cases"])
+    click.echo("Exact match rate: %.3f" % float(result["exact_match_rate"]))
+    click.echo("label\tprecision\trecall\tf1\ttp\tfp\tfn")
+    for label, metrics in result["per_label"].items():
+        click.echo(
+            "%s\t%.3f\t%.3f\t%.3f\t%s\t%s\t%s"
+            % (
+                label,
+                float(metrics["precision"]),
+                float(metrics["recall"]),
+                float(metrics["f1"]),
+                metrics["tp"],
+                metrics["fp"],
+                metrics["fn"],
+            )
+        )
+    missed = [row for row in result["rows"] if not row["exact_match"]]
+    if missed:
+        click.echo("Mismatches:")
+        for row in missed:
+            click.echo(
+                "- %s expected=%s predicted=%s"
+                % (row["id"], ",".join(row["expected"]), ",".join(row["predicted"]))
+            )
+    return 0
+
+
+def _write_verify_report(
+    result: dict,
+    trace: object,
+    *,
+    report: bool,
+    out: Optional[str],
+    default_name: str,
+) -> Optional[str]:
+    if not report and not out:
+        return None
+    output_path = out or str(Path(".contexttrace") / "reports" / default_name)
+    return VerifyReportGenerator().generate(result, trace, path=output_path)
+
+
+def _print_verify_result(
+    result: dict,
+    *,
+    json_output: bool,
+    written_report: Optional[str],
+    fail_on: tuple[str, ...] = (),
+) -> int:
+    fail_messages = _verify_failures(result, fail_on)
+    if json_output:
+        if written_report:
+            click.echo("Report: %s" % written_report, err=True)
+        click.echo(json.dumps(result, indent=2))
+        for message in fail_messages:
+            click.echo("Verification failed: %s" % message, err=True)
+        return 1 if fail_messages else 0
+    summary = result["summary"]
+    click.echo("Claims verified: %s" % summary["total_claims"])
+    click.echo(
+        "Supported: {supported} | Partial: {partially_supported} | Unsupported: {unsupported} | Unverifiable: {unverifiable} | Contradicted: {contradicted}".format(
+            **summary
+        )
+    )
+    click.echo("Support rate: %.3f" % float(summary["support_rate"]))
+    click.echo("Unsupported claim rate: %.3f" % float(summary["unsupported_claim_rate"]))
+    click.echo("Citation mismatches: %s" % summary["citation_mismatches"])
+    click.echo("Failure type: %s" % summary["failure_type"])
+    click.echo("Should abstain: %s" % str(summary["should_abstain"]).lower())
+    click.echo("Suggested fix: %s" % summary["suggested_fix"])
+    if written_report:
+        click.echo("Report: %s" % written_report)
+    for message in fail_messages:
+        click.echo("Verification failed: %s" % message, err=True)
+    return 1 if fail_messages else 0
+
+
+def _verify_failures(result: dict, fail_on: tuple[str, ...]) -> list[str]:
+    if not fail_on:
+        return []
+    summary = result.get("summary") or {}
+    claims = result.get("claims") or []
+    failure_types = set(summary.get("failure_types") or [])
+    messages = []
+    for raw_rule in fail_on:
+        rule = raw_rule.strip().lower().replace("-", "_")
+        if rule == "unsupported" and int(summary.get("unsupported") or 0) > 0:
+            messages.append("unsupported claim detected")
+        elif rule in {"partial", "partial_support", "partially_supported"} and int(summary.get("partially_supported") or 0) > 0:
+            messages.append("partially supported claim detected")
+        elif rule == "citation_mismatch" and "citation_mismatch" in failure_types:
+            messages.append("citation mismatch detected")
+        elif rule == "should_abstain" and bool(summary.get("should_abstain")):
+            messages.append("answer should have abstained")
+        elif rule == "contradicted" and int(summary.get("contradicted") or 0) > 0:
+            messages.append("contradicted claim detected")
+        elif rule == "unverifiable" and int(summary.get("unverifiable") or 0) > 0:
+            messages.append("unverifiable claim detected")
+        elif rule == "no_citation" and any(claim.get("citation_status") == "claim_has_no_citation" for claim in claims):
+            messages.append("claim without citation detected")
+        elif rule == "any_failure" and failure_types != {"no_failure_detected"}:
+            messages.append("verification failure detected")
+        elif rule not in {
+            "unsupported",
+            "partial",
+            "partial_support",
+            "partially_supported",
+            "citation_mismatch",
+            "should_abstain",
+            "contradicted",
+            "unverifiable",
+            "no_citation",
+            "any_failure",
+        }:
+            messages.append("unknown --fail-on rule %s" % raw_rule)
+    return messages
+
+
 @cli.command("eval")
 @click.option("--dataset", required=True, help="Path to eval questions JSON.")
 @click.option("--endpoint", default=None, help="RAG endpoint URL. Defaults to config eval_endpoint.")

contexttrace-0.2.0/contexttrace/verify/__init__.py

@@ -0,0 +1,21 @@
+from contexttrace.verify.runner import verify_trace, verify_trace_file
+from contexttrace.verify.schema import (
+    RAGTrace,
+    TraceCitation,
+    TraceContext,
+    VerificationInputError,
+    load_trace_file,
+)
+from contexttrace.verify.demos import list_verify_demos, load_verify_demo
+
+__all__ = [
+    "RAGTrace",
+    "TraceCitation",
+    "TraceContext",
+    "VerificationInputError",
+    "list_verify_demos",
+    "load_trace_file",
+    "load_verify_demo",
+    "verify_trace",
+    "verify_trace_file",
+]

contexttrace-0.2.0/contexttrace/verify/abstention.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from contexttrace.verify.claims import Claim
+from contexttrace.verify.evidence import find_best_evidence
+from contexttrace.verify.schema import TraceContext
+from contexttrace.verify.verdicts import ClaimVerification
+
+
+def judge_abstention(
+    *,
+    query: str,
+    claims: list[Claim],
+    contexts: list[TraceContext],
+    verifications: list[ClaimVerification],
+    mode: str = "lexical",
+) -> dict[str, object]:
+    if not claims:
+        return {
+            "should_abstain": False,
+            "reason": "The answer does not contain factual claims that require evidence support.",
+        }
+
+    if not contexts:
+        return {
+            "should_abstain": True,
+            "reason": "The answer contains factual claims, but no retrieved contexts were provided.",
+        }
+
+    total_claims = len(verifications)
+    supported = len([item for item in verifications if item.verdict == "supported"])
+    unsupported_like = len(
+        [
+            item
+            for item in verifications
+            if item.verdict in {"unsupported", "contradicted"}
+        ]
+    )
+
+    query_match = find_best_evidence(query, contexts, mode=mode)
+    if supported == 0 and query_match.score < 0.18:
+        return {
+            "should_abstain": True,
+            "reason": (
+                "The query asks for information that does not appear in the retrieved contexts, "
+                "but the answer still gives a factual response."
+            ),
+        }
+
+    if unsupported_like / total_claims >= 0.5:
+        return {
+            "should_abstain": True,
+            "reason": (
+                "The answer contains factual claims, but most important claims are unsupported "
+                "or contradicted by the retrieved contexts."
+            ),
+        }
+
+    if any(item.verdict == "partially_supported" for item in verifications):
+        return {
+            "should_abstain": False,
+            "reason": (
+                "At least one claim is only partially supported; the answer should remove "
+                "or qualify unsupported details rather than fully abstain."
+            ),
+        }
+
+    return {
+        "should_abstain": False,
+        "reason": "Most generated claims are supported by retrieved evidence.",
+    }

contexttrace-0.2.0/contexttrace/verify/benchmark.py

@@ -0,0 +1,229 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from contexttrace.verify.runner import verify_trace
+from contexttrace.verify.schema import RAGTrace, load_trace
+
+
+@dataclass(frozen=True)
+class VerifyBenchmarkCase:
+    id: str
+    trace: RAGTrace
+    expected_labels: set[str]
+
+
+def benchmark_cases() -> list[VerifyBenchmarkCase]:
+    return [
+        _case(
+            "supported_refund_window",
+            {
+                "query": "What is the refund policy?",
+                "answer": "Refunds are allowed within 30 days of purchase.",
+                "contexts": [
+                    {
+                        "id": "policy",
+                        "text": "Customers may request refunds within 30 days of purchase.",
+                    }
+                ],
+                "citations": [
+                    {
+                        "claim": "Refunds are allowed within 30 days of purchase.",
+                        "source_id": "policy",
+                    }
+                ],
+            },
+            {"no_failure_detected"},
+        ),
+        _case(
+            "semantic_money_back_window",
+            {
+                "query": "What is the refund policy?",
+                "answer": "Refunds are allowed within 30 days.",
+                "contexts": [
+                    {
+                        "id": "policy",
+                        "text": "Customers can request money back within thirty days of purchase.",
+                    }
+                ],
+            },
+            {"no_failure_detected"},
+        ),
+        _case(
+            "semantic_order_id",
+            {
+                "query": "What do refund requests need?",
+                "answer": "Refund requests must include an order number.",
+                "contexts": [
+                    {
+                        "id": "policy",
+                        "text": "Refund requests require an order ID.",
+                    }
+                ],
+            },
+            {"no_failure_detected"},
+        ),
+        _case(
+            "unsupported_processing_time",
+            {
+                "query": "How long does refund processing take?",
+                "answer": "Refunds are processed within 5 business days.",
+                "contexts": [
+                    {
+                        "id": "policy",
+                        "text": "Customers may request refunds within 30 days of purchase.",
+                    }
+                ],
+            },
+            {"should_have_abstained", "unsupported_answer"},
+        ),
+        _case(
+            "partial_manager_approval",
+            {
+                "query": "Can customers request refunds within 30 days?",
+                "answer": "Refunds within 30 days require manager approval.",
+                "contexts": [
+                    {
+                        "id": "policy",
+                        "text": "Customers may request refunds within 30 days of purchase.",
+                    }
+                ],
+            },
+            {"partial_support"},
+        ),
+        _case(
+            "citation_wrong_source",
+            {
+                "query": "What is the current refund window?",
+                "answer": "Refunds are allowed within 30 days of purchase.",
+                "contexts": [
+                    {
+                        "id": "archive",
+                        "text": "Customers may exchange eligible items within 14 days of purchase.",
+                    },
+                    {
+                        "id": "current_policy",
+                        "text": "Customers may request refunds within 30 days of purchase.",
+                    },
+                ],
+                "citations": [
+                    {
+                        "claim": "Refunds are allowed within 30 days of purchase.",
+                        "source_id": "archive",
+                    }
+                ],
+            },
+            {"citation_mismatch"},
+        ),
+        _case(
+            "should_abstain_vip",
+            {
+                "query": "What refund exception applies to VIP customers?",
+                "answer": "VIP customers receive cash refunds up to 90 days after purchase.",
+                "contexts": [
+                    {
+                        "id": "shipping",
+                        "text": "Standard shipping takes 3 to 5 business days.",
+                    }
+                ],
+            },
+            {"should_have_abstained", "unsupported_answer"},
+        ),
+        _case(
+            "contradicted_refund_allowed",
+            {
+                "query": "Are refunds allowed within 30 days?",
+                "answer": "Refunds are allowed within 30 days of purchase.",
+                "contexts": [
+                    {
+                        "id": "policy",
+                        "text": "Refunds are not allowed within 30 days of purchase.",
+                    }
+                ],
+            },
+            {"should_have_abstained", "contradicted_answer"},
+        ),
+        _case(
+            "supported_two_claims",
+            {
+                "query": "What is the refund policy?",
+                "answer": "Refunds are allowed within 30 days of purchase. Refund requests must include an order number.",
+                "contexts": [
+                    {
+                        "id": "policy",
+                        "text": "Customers may request refunds within 30 days of purchase. Refund requests must include an order number.",
+                    }
+                ],
+                "citations": [
+                    {
+                        "claim": "Refunds are allowed within 30 days of purchase.",
+                        "source_id": "policy",
+                    },
+                    {
+                        "claim": "Refund requests must include an order number.",
+                        "source_id": "policy",
+                    },
+                ],
+            },
+            {"no_failure_detected"},
+        ),
+    ]
+
+
+def run_verify_benchmark(*, mode: str = "lexical") -> dict[str, Any]:
+    rows = []
+    labels = set()
+    for case in benchmark_cases():
+        result = verify_trace(case.trace, mode=mode)
+        predicted = _predicted_labels(result)
+        labels.update(case.expected_labels)
+        labels.update(predicted)
+        rows.append(
+            {
+                "id": case.id,
+                "expected": sorted(case.expected_labels),
+                "predicted": sorted(predicted),
+                "exact_match": predicted == case.expected_labels,
+                "summary": result.get("summary") or {},
+            }
+        )
+
+    per_label = {}
+    for label in sorted(labels):
+        tp = sum(1 for row in rows if label in row["expected"] and label in row["predicted"])
+        fp = sum(1 for row in rows if label not in row["expected"] and label in row["predicted"])
+        fn = sum(1 for row in rows if label in row["expected"] and label not in row["predicted"])
+        precision = tp / (tp + fp) if tp + fp else 0.0
+        recall = tp / (tp + fn) if tp + fn else 0.0
+        f1 = (2 * precision * recall / (precision + recall)) if precision + recall else 0.0
+        per_label[label] = {
+            "tp": tp,
+            "fp": fp,
+            "fn": fn,
+            "precision": round(precision, 3),
+            "recall": round(recall, 3),
+            "f1": round(f1, 3),
+        }
+
+    exact_matches = sum(1 for row in rows if row["exact_match"])
+    return {
+        "mode": mode,
+        "cases": len(rows),
+        "exact_match_rate": round(exact_matches / len(rows), 3) if rows else 0.0,
+        "per_label": per_label,
+        "rows": rows,
+    }
+
+
+def _case(case_id: str, payload: dict[str, Any], expected_labels: set[str]) -> VerifyBenchmarkCase:
+    return VerifyBenchmarkCase(
+        id=case_id,
+        trace=load_trace(payload, source="benchmark case %s" % case_id),
+        expected_labels=expected_labels,
+    )
+
+
+def _predicted_labels(result: dict[str, Any]) -> set[str]:
+    labels = set((result.get("summary") or {}).get("failure_types") or [])
+    return labels or {"no_failure_detected"}