contexttrace 0.2.0__tar.gz → 0.4.0__tar.gz

{contexttrace-0.2.0 → contexttrace-0.4.0}/MANIFEST.in

@@ -3,6 +3,7 @@ include pyproject.toml
 include setup.py
 include contexttrace/py.typed
 recursive-include contexttrace *.py
+recursive-include contexttrace/verify *.json
 prune build
 prune dist
 prune tests

{contexttrace-0.2.0 → contexttrace-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: contexttrace
-Version: 0.2.0
+Version: 0.4.0
 Summary: Local-first SDK and CLI for RAG and agent reliability tracing, citation checks, and failure diagnosis.
 Author: ContextTrace contributors
 License: MIT
@@ -145,17 +145,26 @@ contexttrace verify trace.json --report --out reports/example.html
 contexttrace verify trace.json --mode semantic
 contexttrace verify trace.json --fail-on unsupported --fail-on citation_mismatch
 contexttrace verify-benchmark --mode semantic
+contexttrace verify-benchmark --mode semantic --report
+contexttrace verify-benchmark --case-set external --mode semantic --report
+contexttrace compare baseline.json current.json
+contexttrace compare baseline.json current.json --report
+contexttrace compare baseline.json current.json --fail-on new_failure
 ```
 
 Input requires `query`, `answer`, and `contexts` with `id` and `text`. Optional `citations` are checked to catch cited sources that do not actually support the matched claim.
 
 `verify-demo` uses bundled demo traces, so it works immediately after `pip install contexttrace`. Available demos include `unsupported_claim`, `partial_support`, `citation_mismatch`, `should_abstain`, and `supported_answer`.
 
-Use `--mode semantic` for local paraphrase-aware matching, and `verify-benchmark` to inspect bundled precision/recall metrics.
+Use `--mode semantic` for local paraphrase-aware matching, and `verify-benchmark` to inspect bundled precision/recall metrics. The default benchmark includes 32 real ContextTrace docs and release-artifact cases. `--case-set external` adds public OSS documentation and GitHub issue cases from Qdrant, Chroma, Haystack, and LangChain, while `--case-set all` runs both packs. `--report` writes an HTML report with misses to inspect.
 
-ContextTrace verifies whether each generated claim is actually supported by retrieved evidence. Instead of only showing a trace or a score, it tells you where the evidence chain broke: unsupported claim, citation mismatch, insufficient context, or should-have-abstained.
+Verification output includes evidence span offsets, stable span hashes, multiple supporting spans, typed matched/missing facts, and claim-level root causes so partial support failures are easier to inspect.
 
-The v0.2.0 verifier uses local lexical heuristics by default. Claim extraction is rule-based, contradiction detection is conservative, and semantic or LLM-judge support can be added later.
+ContextTrace verifies whether each generated claim is actually supported by retrieved evidence. Instead of only showing a trace or a score, it tells you where the evidence chain broke: unsupported claim, citation mismatch, retrieval miss, answer overreach, conflicting context, or should-have-abstained.
+
+Use `contexttrace compare baseline.json current.json` to diff two portable traces or saved `verify --json` outputs. It reports support-rate deltas, new unsupported claims, citation regressions, should-abstain flips, and new root causes, with `--fail-on` gates for CI.
+
+The v0.4.0 verifier uses local lexical heuristics by default. Claim extraction is rule-based, contradiction detection is conservative, and semantic or LLM-judge support can be added later.
 
 ## What It Catches
 

{contexttrace-0.2.0 → contexttrace-0.4.0}/README.md

@@ -88,17 +88,26 @@ contexttrace verify trace.json --report --out reports/example.html
 contexttrace verify trace.json --mode semantic
 contexttrace verify trace.json --fail-on unsupported --fail-on citation_mismatch
 contexttrace verify-benchmark --mode semantic
+contexttrace verify-benchmark --mode semantic --report
+contexttrace verify-benchmark --case-set external --mode semantic --report
+contexttrace compare baseline.json current.json
+contexttrace compare baseline.json current.json --report
+contexttrace compare baseline.json current.json --fail-on new_failure
 ```
 
 Input requires `query`, `answer`, and `contexts` with `id` and `text`. Optional `citations` are checked to catch cited sources that do not actually support the matched claim.
 
 `verify-demo` uses bundled demo traces, so it works immediately after `pip install contexttrace`. Available demos include `unsupported_claim`, `partial_support`, `citation_mismatch`, `should_abstain`, and `supported_answer`.
 
-Use `--mode semantic` for local paraphrase-aware matching, and `verify-benchmark` to inspect bundled precision/recall metrics.
+Use `--mode semantic` for local paraphrase-aware matching, and `verify-benchmark` to inspect bundled precision/recall metrics. The default benchmark includes 32 real ContextTrace docs and release-artifact cases. `--case-set external` adds public OSS documentation and GitHub issue cases from Qdrant, Chroma, Haystack, and LangChain, while `--case-set all` runs both packs. `--report` writes an HTML report with misses to inspect.
 
-ContextTrace verifies whether each generated claim is actually supported by retrieved evidence. Instead of only showing a trace or a score, it tells you where the evidence chain broke: unsupported claim, citation mismatch, insufficient context, or should-have-abstained.
+Verification output includes evidence span offsets, stable span hashes, multiple supporting spans, typed matched/missing facts, and claim-level root causes so partial support failures are easier to inspect.
 
-The v0.2.0 verifier uses local lexical heuristics by default. Claim extraction is rule-based, contradiction detection is conservative, and semantic or LLM-judge support can be added later.
+ContextTrace verifies whether each generated claim is actually supported by retrieved evidence. Instead of only showing a trace or a score, it tells you where the evidence chain broke: unsupported claim, citation mismatch, retrieval miss, answer overreach, conflicting context, or should-have-abstained.
+
+Use `contexttrace compare baseline.json current.json` to diff two portable traces or saved `verify --json` outputs. It reports support-rate deltas, new unsupported claims, citation regressions, should-abstain flips, and new root causes, with `--fail-on` gates for CI.
+
+The v0.4.0 verifier uses local lexical heuristics by default. Claim extraction is rule-based, contradiction detection is conservative, and semantic or LLM-judge support can be added later.
 
 ## What It Catches
 

contexttrace-0.4.0/contexttrace/_version.py

@@ -0,0 +1 @@
+__version__ = "0.4.0"

{contexttrace-0.2.0 → contexttrace-0.4.0}/contexttrace/cli.py

@@ -24,12 +24,15 @@ from contexttrace.storage import SQLiteTraceStore
 from contexttrace.thresholds import parse_thresholds, threshold_failures
 from contexttrace.verify import (
     VerificationInputError,
+    compare_failures,
+    compare_trace_files,
     list_verify_demos,
     load_trace_file,
     load_verify_demo,
     verify_trace,
 )
-from contexttrace.verify.benchmark import run_verify_benchmark
+from contexttrace.verify.benchmark import run_verify_benchmark, write_verify_benchmark_report
+from contexttrace.verify.compare_report import CompareReportGenerator
 from contexttrace.verify.report import VerifyReportGenerator
 from contexttrace.viewer import serve_viewer
 
@@ -288,18 +291,31 @@ def verify_demo_command(
 
 @cli.command("verify-benchmark")
 @click.option("--mode", default="lexical", show_default=True, type=click.Choice(["lexical", "semantic"]), help="Evidence scoring mode.")
+@click.option("--case-set", default="contexttrace", show_default=True, type=click.Choice(["contexttrace", "external", "all"]), help="Benchmark case set to run.")
 @click.option("--json", "json_output", is_flag=True, help="Print benchmark results as JSON.")
-def verify_benchmark_command(mode: str, json_output: bool) -> int:
+@click.option("--report", is_flag=True, help="Generate a local HTML benchmark report.")
+@click.option("--out", default=None, help="HTML benchmark report path. Implies --report when provided.")
+def verify_benchmark_command(mode: str, case_set: str, json_output: bool, report: bool, out: Optional[str]) -> int:
     """Run the bundled verification precision/recall benchmark."""
 
-    result = run_verify_benchmark(mode=mode)
+    result = run_verify_benchmark(mode=mode, case_set=case_set)
+    written_report = None
+    if report or out:
+        output_path = out or str(Path(".contexttrace") / "reports" / ("verify_benchmark_%s.html" % mode))
+        written_report = write_verify_benchmark_report(result, path=output_path)
     if json_output:
+        if written_report:
+            click.echo("Report: %s" % written_report, err=True)
         click.echo(json.dumps(result, indent=2))
         return 0
 
     click.echo("Mode: %s" % result["mode"])
+    click.echo("Case source: %s" % result["case_source"])
     click.echo("Cases: %s" % result["cases"])
     click.echo("Exact match rate: %.3f" % float(result["exact_match_rate"]))
+    click.echo("Verdict match rate: %.3f" % float(result["verdict_match_rate"]))
+    click.echo("Citation match rate: %.3f" % float(result["citation_match_rate"]))
+    click.echo("Abstention match rate: %.3f" % float(result["abstention_match_rate"]))
     click.echo("label\tprecision\trecall\tf1\ttp\tfp\tfn")
     for label, metrics in result["per_label"].items():
         click.echo(
@@ -322,9 +338,72 @@ def verify_benchmark_command(mode: str, json_output: bool) -> int:
                 "- %s expected=%s predicted=%s"
                 % (row["id"], ",".join(row["expected"]), ",".join(row["predicted"]))
             )
+    if written_report:
+        click.echo("Report: %s" % written_report)
     return 0
 
 
+@cli.command("compare")
+@click.argument("baseline_json")
+@click.argument("current_json")
+@click.option("--json", "json_output", is_flag=True, help="Print the full comparison result as JSON.")
+@click.option("--report", is_flag=True, help="Generate a local HTML regression report.")
+@click.option("--out", default=None, help="HTML report path. Implies --report when provided.")
+@click.option("--mode", default="lexical", show_default=True, type=click.Choice(["lexical", "semantic"]), help="Evidence scoring mode for raw trace inputs.")
+@click.option("--fail-on", multiple=True, help="Fail on new_failure, new_unsupported, new_citation_mismatch, should_abstain_flip, support_rate_drop, new_root_cause, or any_regression.")
+def compare_command(
+    baseline_json: str,
+    current_json: str,
+    json_output: bool,
+    report: bool,
+    out: Optional[str],
+    mode: str,
+    fail_on: tuple[str, ...],
+) -> int:
+    """Compare two portable RAG traces or verification JSON outputs."""
+
+    try:
+        result = compare_trace_files(baseline_json, current_json, mode=mode)
+    except VerificationInputError as exc:
+        raise click.ClickException(str(exc)) from exc
+
+    written_report = None
+    if report or out:
+        default_name = "%s_vs_%s_compare.html" % (Path(baseline_json).stem, Path(current_json).stem)
+        output_path = out or str(Path(".contexttrace") / "reports" / default_name)
+        written_report = CompareReportGenerator().generate(result, path=output_path)
+
+    fail_messages = compare_failures(result, fail_on)
+    if json_output:
+        if written_report:
+            click.echo("Report: %s" % written_report, err=True)
+        click.echo(json.dumps(result, indent=2))
+        for message in fail_messages:
+            click.echo("Comparison failed: %s" % message, err=True)
+        return 1 if fail_messages else 0
+
+    summary = result["summary"]
+    click.echo("Regression: %s" % str(summary["regression"]).lower())
+    click.echo("Support rate: %.3f -> %.3f (%+.3f)" % (
+        float(summary.get("support_rate_before") or 0.0),
+        float(summary.get("support_rate_after") or 0.0),
+        float(summary.get("support_rate_delta") or 0.0),
+    ))
+    click.echo("Unsupported claim rate delta: %+.3f" % float(summary.get("unsupported_claim_rate_delta") or 0.0))
+    click.echo("Citation mismatch delta: %+d" % int(summary.get("citation_mismatch_delta") or 0))
+    click.echo("New failures: %s" % summary["new_failures"])
+    click.echo("Resolved failures: %s" % summary["resolved_failures"])
+    click.echo("Added claims: %s" % summary["added_claims"])
+    click.echo("Removed claims: %s" % summary["removed_claims"])
+    click.echo("Changed claims: %s" % summary["changed_claims"])
+    click.echo("New root causes: %s" % (", ".join(summary.get("new_root_causes") or []) or "none"))
+    if written_report:
+        click.echo("Report: %s" % written_report)
+    for message in fail_messages:
+        click.echo("Comparison failed: %s" % message, err=True)
+    return 1 if fail_messages else 0
+
+
 def _write_verify_report(
     result: dict,
     trace: object,
@@ -365,6 +444,7 @@ def _print_verify_result(
     click.echo("Unsupported claim rate: %.3f" % float(summary["unsupported_claim_rate"]))
     click.echo("Citation mismatches: %s" % summary["citation_mismatches"])
     click.echo("Failure type: %s" % summary["failure_type"])
+    click.echo("Primary root cause: %s" % summary.get("primary_root_cause", "unknown"))
     click.echo("Should abstain: %s" % str(summary["should_abstain"]).lower())
     click.echo("Suggested fix: %s" % summary["suggested_fix"])
     if written_report:

{contexttrace-0.2.0 → contexttrace-0.4.0}/contexttrace/verify/__init__.py

@@ -1,4 +1,5 @@
 from contexttrace.verify.runner import verify_trace, verify_trace_file
+from contexttrace.verify.compare import compare_failures, compare_trace_files, compare_verifications
 from contexttrace.verify.schema import (
     RAGTrace,
     TraceCitation,
@@ -13,6 +14,9 @@ __all__ = [
     "TraceCitation",
     "TraceContext",
     "VerificationInputError",
+    "compare_failures",
+    "compare_trace_files",
+    "compare_verifications",
     "list_verify_demos",
     "load_trace_file",
     "load_verify_demo",