contexttrace 0.1.0__tar.gz → 0.3.0__tar.gz

{contexttrace-0.1.0 → contexttrace-0.3.0}/MANIFEST.in

@@ -3,6 +3,7 @@ include pyproject.toml
 include setup.py
 include contexttrace/py.typed
 recursive-include contexttrace *.py
+recursive-include contexttrace/verify *.json
 prune build
 prune dist
 prune tests

{contexttrace-0.1.0 → contexttrace-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: contexttrace
-Version: 0.1.0
+Version: 0.3.0
 Summary: Local-first SDK and CLI for RAG and agent reliability tracing, citation checks, and failure diagnosis.
 Author: ContextTrace contributors
 License: MIT
@@ -133,6 +133,34 @@ contexttrace eval \
   --fail-on "failure_rate>0.25"
 ```
 
+## Claim-Level Evidence Verification
+
+Verify a portable RAG trace artifact without a hosted dashboard:
+
+```bash
+contexttrace verify-demo unsupported_claim --report
+contexttrace verify trace.json
+contexttrace verify trace.json --json
+contexttrace verify trace.json --report --out reports/example.html
+contexttrace verify trace.json --mode semantic
+contexttrace verify trace.json --fail-on unsupported --fail-on citation_mismatch
+contexttrace verify-benchmark --mode semantic
+contexttrace verify-benchmark --mode semantic --report
+contexttrace verify-benchmark --case-set external --mode semantic --report
+```
+
+Input requires `query`, `answer`, and `contexts` with `id` and `text`. Optional `citations` are checked to catch cited sources that do not actually support the matched claim.
+
+`verify-demo` uses bundled demo traces, so it works immediately after `pip install contexttrace`. Available demos include `unsupported_claim`, `partial_support`, `citation_mismatch`, `should_abstain`, and `supported_answer`.
+
+Use `--mode semantic` for local paraphrase-aware matching, and `verify-benchmark` to inspect bundled precision/recall metrics. The default benchmark includes 32 real ContextTrace docs and release-artifact cases. `--case-set external` adds public OSS documentation and GitHub issue cases from Qdrant, Chroma, Haystack, and LangChain, while `--case-set all` runs both packs. `--report` writes an HTML report with misses to inspect.
+
+Verification output includes evidence span offsets, stable span hashes, multiple supporting spans, typed matched/missing facts, and claim-level root causes so partial support failures are easier to inspect.
+
+ContextTrace verifies whether each generated claim is actually supported by retrieved evidence. Instead of only showing a trace or a score, it tells you where the evidence chain broke: unsupported claim, citation mismatch, retrieval miss, answer overreach, conflicting context, or should-have-abstained.
+
+The v0.3.0 verifier uses local lexical heuristics by default. Claim extraction is rule-based, contradiction detection is conservative, and semantic or LLM-judge support can be added later.
+
 ## What It Catches
 
 - `retrieval_miss`

{contexttrace-0.1.0 → contexttrace-0.3.0}/README.md

@@ -76,6 +76,34 @@ contexttrace eval \
   --fail-on "failure_rate>0.25"
 ```
 
+## Claim-Level Evidence Verification
+
+Verify a portable RAG trace artifact without a hosted dashboard:
+
+```bash
+contexttrace verify-demo unsupported_claim --report
+contexttrace verify trace.json
+contexttrace verify trace.json --json
+contexttrace verify trace.json --report --out reports/example.html
+contexttrace verify trace.json --mode semantic
+contexttrace verify trace.json --fail-on unsupported --fail-on citation_mismatch
+contexttrace verify-benchmark --mode semantic
+contexttrace verify-benchmark --mode semantic --report
+contexttrace verify-benchmark --case-set external --mode semantic --report
+```
+
+Input requires `query`, `answer`, and `contexts` with `id` and `text`. Optional `citations` are checked to catch cited sources that do not actually support the matched claim.
+
+`verify-demo` uses bundled demo traces, so it works immediately after `pip install contexttrace`. Available demos include `unsupported_claim`, `partial_support`, `citation_mismatch`, `should_abstain`, and `supported_answer`.
+
+Use `--mode semantic` for local paraphrase-aware matching, and `verify-benchmark` to inspect bundled precision/recall metrics. The default benchmark includes 32 real ContextTrace docs and release-artifact cases. `--case-set external` adds public OSS documentation and GitHub issue cases from Qdrant, Chroma, Haystack, and LangChain, while `--case-set all` runs both packs. `--report` writes an HTML report with misses to inspect.
+
+Verification output includes evidence span offsets, stable span hashes, multiple supporting spans, typed matched/missing facts, and claim-level root causes so partial support failures are easier to inspect.
+
+ContextTrace verifies whether each generated claim is actually supported by retrieved evidence. Instead of only showing a trace or a score, it tells you where the evidence chain broke: unsupported claim, citation mismatch, retrieval miss, answer overreach, conflicting context, or should-have-abstained.
+
+The v0.3.0 verifier uses local lexical heuristics by default. Claim extraction is rule-based, contradiction detection is conservative, and semantic or LLM-judge support can be added later.
+
 ## What It Catches
 
 - `retrieval_miss`

contexttrace-0.3.0/contexttrace/_version.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

{contexttrace-0.1.0 → contexttrace-0.3.0}/contexttrace/cli.py

@@ -22,6 +22,15 @@ from contexttrace.regression import BENCHMARK_STRATEGIES, run_local_benchmark
 from contexttrace.report import ReportGenerator
 from contexttrace.storage import SQLiteTraceStore
 from contexttrace.thresholds import parse_thresholds, threshold_failures
+from contexttrace.verify import (
+    VerificationInputError,
+    list_verify_demos,
+    load_trace_file,
+    load_verify_demo,
+    verify_trace,
+)
+from contexttrace.verify.benchmark import run_verify_benchmark, write_verify_benchmark_report
+from contexttrace.verify.report import VerifyReportGenerator
 from contexttrace.viewer import serve_viewer
 
 
@@ -198,6 +207,230 @@ def report(
         webbrowser.open(Path(written).resolve().as_uri())
 
 
+@cli.command("verify")
+@click.argument("trace_json")
+@click.option("--json", "json_output", is_flag=True, help="Print the full verification result as JSON.")
+@click.option("--report", is_flag=True, help="Generate a local HTML verification report.")
+@click.option("--out", default=None, help="HTML report path. Implies --report when provided.")
+@click.option("--mode", default="lexical", show_default=True, type=click.Choice(["lexical", "semantic"]), help="Evidence scoring mode.")
+@click.option("--fail-on", multiple=True, help="Fail on unsupported, partial_support, citation_mismatch, should_abstain, contradicted, unverifiable, no_citation, or any_failure.")
+def verify_command(
+    trace_json: str,
+    json_output: bool,
+    report: bool,
+    out: Optional[str],
+    mode: str,
+    fail_on: tuple[str, ...],
+) -> int:
+    """Verify claim-level evidence support for a portable RAG trace JSON file."""
+
+    try:
+        trace = load_trace_file(trace_json)
+    except VerificationInputError as exc:
+        raise click.ClickException(str(exc)) from exc
+
+    result = verify_trace(trace, mode=mode)
+    written_report = _write_verify_report(
+        result,
+        trace,
+        report=report,
+        out=out,
+        default_name="%s_verify.html" % Path(trace_json).stem,
+    )
+    return _print_verify_result(
+        result,
+        json_output=json_output,
+        written_report=written_report,
+        fail_on=fail_on,
+    )
+
+
+@cli.command("verify-demo")
+@click.argument("demo_name", required=False, default="unsupported_claim")
+@click.option("--json", "json_output", is_flag=True, help="Print the full verification result as JSON.")
+@click.option("--report", is_flag=True, help="Generate a local HTML verification report.")
+@click.option("--out", default=None, help="HTML report path. Implies --report when provided.")
+@click.option("--mode", default="lexical", show_default=True, type=click.Choice(["lexical", "semantic"]), help="Evidence scoring mode.")
+@click.option("--fail-on", multiple=True, help="Fail on unsupported, partial_support, citation_mismatch, should_abstain, contradicted, unverifiable, no_citation, or any_failure.")
+def verify_demo_command(
+    demo_name: str,
+    json_output: bool,
+    report: bool,
+    out: Optional[str],
+    mode: str,
+    fail_on: tuple[str, ...],
+) -> int:
+    """Run a bundled claim-level verification demo."""
+
+    try:
+        trace = load_verify_demo(demo_name)
+    except KeyError as exc:
+        raise click.ClickException(
+            "Unknown verify demo %s. Available demos: %s"
+            % (demo_name, ", ".join(list_verify_demos()))
+        ) from exc
+
+    result = verify_trace(trace, mode=mode)
+    written_report = _write_verify_report(
+        result,
+        trace,
+        report=report,
+        out=out,
+        default_name="%s_verify_demo.html" % demo_name,
+    )
+    return _print_verify_result(
+        result,
+        json_output=json_output,
+        written_report=written_report,
+        fail_on=fail_on,
+    )
+
+
+@cli.command("verify-benchmark")
+@click.option("--mode", default="lexical", show_default=True, type=click.Choice(["lexical", "semantic"]), help="Evidence scoring mode.")
+@click.option("--case-set", default="contexttrace", show_default=True, type=click.Choice(["contexttrace", "external", "all"]), help="Benchmark case set to run.")
+@click.option("--json", "json_output", is_flag=True, help="Print benchmark results as JSON.")
+@click.option("--report", is_flag=True, help="Generate a local HTML benchmark report.")
+@click.option("--out", default=None, help="HTML benchmark report path. Implies --report when provided.")
+def verify_benchmark_command(mode: str, case_set: str, json_output: bool, report: bool, out: Optional[str]) -> int:
+    """Run the bundled verification precision/recall benchmark."""
+
+    result = run_verify_benchmark(mode=mode, case_set=case_set)
+    written_report = None
+    if report or out:
+        output_path = out or str(Path(".contexttrace") / "reports" / ("verify_benchmark_%s.html" % mode))
+        written_report = write_verify_benchmark_report(result, path=output_path)
+    if json_output:
+        if written_report:
+            click.echo("Report: %s" % written_report, err=True)
+        click.echo(json.dumps(result, indent=2))
+        return 0
+
+    click.echo("Mode: %s" % result["mode"])
+    click.echo("Case source: %s" % result["case_source"])
+    click.echo("Cases: %s" % result["cases"])
+    click.echo("Exact match rate: %.3f" % float(result["exact_match_rate"]))
+    click.echo("Verdict match rate: %.3f" % float(result["verdict_match_rate"]))
+    click.echo("Citation match rate: %.3f" % float(result["citation_match_rate"]))
+    click.echo("Abstention match rate: %.3f" % float(result["abstention_match_rate"]))
+    click.echo("label\tprecision\trecall\tf1\ttp\tfp\tfn")
+    for label, metrics in result["per_label"].items():
+        click.echo(
+            "%s\t%.3f\t%.3f\t%.3f\t%s\t%s\t%s"
+            % (
+                label,
+                float(metrics["precision"]),
+                float(metrics["recall"]),
+                float(metrics["f1"]),
+                metrics["tp"],
+                metrics["fp"],
+                metrics["fn"],
+            )
+        )
+    missed = [row for row in result["rows"] if not row["exact_match"]]
+    if missed:
+        click.echo("Mismatches:")
+        for row in missed:
+            click.echo(
+                "- %s expected=%s predicted=%s"
+                % (row["id"], ",".join(row["expected"]), ",".join(row["predicted"]))
+            )
+    if written_report:
+        click.echo("Report: %s" % written_report)
+    return 0
+
+
+def _write_verify_report(
+    result: dict,
+    trace: object,
+    *,
+    report: bool,
+    out: Optional[str],
+    default_name: str,
+) -> Optional[str]:
+    if not report and not out:
+        return None
+    output_path = out or str(Path(".contexttrace") / "reports" / default_name)
+    return VerifyReportGenerator().generate(result, trace, path=output_path)
+
+
+def _print_verify_result(
+    result: dict,
+    *,
+    json_output: bool,
+    written_report: Optional[str],
+    fail_on: tuple[str, ...] = (),
+) -> int:
+    fail_messages = _verify_failures(result, fail_on)
+    if json_output:
+        if written_report:
+            click.echo("Report: %s" % written_report, err=True)
+        click.echo(json.dumps(result, indent=2))
+        for message in fail_messages:
+            click.echo("Verification failed: %s" % message, err=True)
+        return 1 if fail_messages else 0
+    summary = result["summary"]
+    click.echo("Claims verified: %s" % summary["total_claims"])
+    click.echo(
+        "Supported: {supported} | Partial: {partially_supported} | Unsupported: {unsupported} | Unverifiable: {unverifiable} | Contradicted: {contradicted}".format(
+            **summary
+        )
+    )
+    click.echo("Support rate: %.3f" % float(summary["support_rate"]))
+    click.echo("Unsupported claim rate: %.3f" % float(summary["unsupported_claim_rate"]))
+    click.echo("Citation mismatches: %s" % summary["citation_mismatches"])
+    click.echo("Failure type: %s" % summary["failure_type"])
+    click.echo("Primary root cause: %s" % summary.get("primary_root_cause", "unknown"))
+    click.echo("Should abstain: %s" % str(summary["should_abstain"]).lower())
+    click.echo("Suggested fix: %s" % summary["suggested_fix"])
+    if written_report:
+        click.echo("Report: %s" % written_report)
+    for message in fail_messages:
+        click.echo("Verification failed: %s" % message, err=True)
+    return 1 if fail_messages else 0
+
+
+def _verify_failures(result: dict, fail_on: tuple[str, ...]) -> list[str]:
+    if not fail_on:
+        return []
+    summary = result.get("summary") or {}
+    claims = result.get("claims") or []
+    failure_types = set(summary.get("failure_types") or [])
+    messages = []
+    for raw_rule in fail_on:
+        rule = raw_rule.strip().lower().replace("-", "_")
+        if rule == "unsupported" and int(summary.get("unsupported") or 0) > 0:
+            messages.append("unsupported claim detected")
+        elif rule in {"partial", "partial_support", "partially_supported"} and int(summary.get("partially_supported") or 0) > 0:
+            messages.append("partially supported claim detected")
+        elif rule == "citation_mismatch" and "citation_mismatch" in failure_types:
+            messages.append("citation mismatch detected")
+        elif rule == "should_abstain" and bool(summary.get("should_abstain")):
+            messages.append("answer should have abstained")
+        elif rule == "contradicted" and int(summary.get("contradicted") or 0) > 0:
+            messages.append("contradicted claim detected")
+        elif rule == "unverifiable" and int(summary.get("unverifiable") or 0) > 0:
+            messages.append("unverifiable claim detected")
+        elif rule == "no_citation" and any(claim.get("citation_status") == "claim_has_no_citation" for claim in claims):
+            messages.append("claim without citation detected")
+        elif rule == "any_failure" and failure_types != {"no_failure_detected"}:
+            messages.append("verification failure detected")
+        elif rule not in {
+            "unsupported",
+            "partial",
+            "partial_support",
+            "partially_supported",
+            "citation_mismatch",
+            "should_abstain",
+            "contradicted",
+            "unverifiable",
+            "no_citation",
+            "any_failure",
+        }:
+            messages.append("unknown --fail-on rule %s" % raw_rule)
+    return messages
+
+
 @cli.command("eval")
 @click.option("--dataset", required=True, help="Path to eval questions JSON.")
 @click.option("--endpoint", default=None, help="RAG endpoint URL. Defaults to config eval_endpoint.")

contexttrace-0.3.0/contexttrace/verify/__init__.py

@@ -0,0 +1,21 @@
+from contexttrace.verify.runner import verify_trace, verify_trace_file
+from contexttrace.verify.schema import (
+    RAGTrace,
+    TraceCitation,
+    TraceContext,
+    VerificationInputError,
+    load_trace_file,
+)
+from contexttrace.verify.demos import list_verify_demos, load_verify_demo
+
+__all__ = [
+    "RAGTrace",
+    "TraceCitation",
+    "TraceContext",
+    "VerificationInputError",
+    "list_verify_demos",
+    "load_trace_file",
+    "load_verify_demo",
+    "verify_trace",
+    "verify_trace_file",
+]

contexttrace-0.3.0/contexttrace/verify/abstention.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from contexttrace.verify.claims import Claim
+from contexttrace.verify.evidence import find_best_evidence
+from contexttrace.verify.schema import TraceContext
+from contexttrace.verify.verdicts import ClaimVerification
+
+
+def judge_abstention(
+    *,
+    query: str,
+    claims: list[Claim],
+    contexts: list[TraceContext],
+    verifications: list[ClaimVerification],
+    mode: str = "lexical",
+) -> dict[str, object]:
+    if not claims:
+        return {
+            "should_abstain": False,
+            "reason": "The answer does not contain factual claims that require evidence support.",
+        }
+
+    if not contexts:
+        return {
+            "should_abstain": True,
+            "reason": "The answer contains factual claims, but no retrieved contexts were provided.",
+        }
+
+    total_claims = len(verifications)
+    supported = len([item for item in verifications if item.verdict == "supported"])
+    unsupported_like = len(
+        [
+            item
+            for item in verifications
+            if item.verdict in {"unsupported", "contradicted"}
+        ]
+    )
+
+    query_match = find_best_evidence(query, contexts, mode=mode)
+    if supported == 0 and query_match.score < 0.18:
+        return {
+            "should_abstain": True,
+            "reason": (
+                "The query asks for information that does not appear in the retrieved contexts, "
+                "but the answer still gives a factual response."
+            ),
+        }
+
+    if unsupported_like / total_claims >= 0.5:
+        return {
+            "should_abstain": True,
+            "reason": (
+                "The answer contains factual claims, but most important claims are unsupported "
+                "or contradicted by the retrieved contexts."
+            ),
+        }
+
+    if any(item.verdict == "partially_supported" for item in verifications):
+        return {
+            "should_abstain": False,
+            "reason": (
+                "At least one claim is only partially supported; the answer should remove "
+                "or qualify unsupported details rather than fully abstain."
+            ),
+        }
+
+    return {
+        "should_abstain": False,
+        "reason": "Most generated claims are supported by retrieved evidence.",
+    }