trace2eval-cli 0.2.1__py3-none-any.whl

trace2eval/__init__.py

@@ -0,0 +1,65 @@
+"""trace2eval -- turn production LLM traces into a regression eval set.
+
+The gap this fills: tracing tools record what happened, and eval frameworks
+score cases you already wrote. Nothing connects the two. The step in between --
+deciding which recorded calls deserve to become permanent test cases -- is still
+done by hand, by scrolling through logs.
+
+    from trace2eval import load_traces, select_cases
+
+    loaded = load_traces("traces.jsonl")
+    result = select_cases(loaded.traces, max_cases=25)
+    for case in result.cases:
+        print(case["id"], case["score"], case["input"])
+
+Everything here is deterministic and offline. No model is called, no API key is
+needed, and the same log always produces the same case set.
+"""
+
+from .checks import CheckResult, detect_fallback, run_check, run_checks
+from .matchers import char_bigram_jaccard, word_jaccard
+from .runner import RunMetrics, compare_runs, load_outputs, run_cases
+from .schema import Trace, load_traces, parse_trace
+from .select import (
+    DEFAULT_DEDUP_THRESHOLD,
+    Matcher,
+    MatcherSpecError,
+    build_case,
+    load_matcher,
+    overlap_coefficient,
+    select_cases,
+    shingle_overlap,
+)
+from .signals import DEFAULT_WEIGHTS, Signal, build_context, compute_signals, score
+
+__version__ = "0.2.1"
+
+__all__ = [
+    "__version__",
+    "CheckResult",
+    "DEFAULT_DEDUP_THRESHOLD",
+    "DEFAULT_WEIGHTS",
+    "Matcher",
+    "MatcherSpecError",
+    "RunMetrics",
+    "Signal",
+    "Trace",
+    "build_case",
+    "build_context",
+    "char_bigram_jaccard",
+    "compare_runs",
+    "compute_signals",
+    "detect_fallback",
+    "load_matcher",
+    "load_outputs",
+    "load_traces",
+    "overlap_coefficient",
+    "parse_trace",
+    "run_check",
+    "run_checks",
+    "run_cases",
+    "score",
+    "select_cases",
+    "shingle_overlap",
+    "word_jaccard",
+]

trace2eval/__main__.py

@@ -0,0 +1,10 @@
+"""Allow ``python -m trace2eval`` as an alternative to the console script."""
+
+from __future__ import annotations
+
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

trace2eval/checks.py

@@ -0,0 +1,149 @@
+"""Deterministic checks.
+
+Every check in this module is a pure function of the output string. No model is
+called, nothing is paid for, and the same input always produces the same verdict.
+
+That constraint is a feature, not a shortcut: a regression gate that depends on
+a paid judge model is a gate that people disable the moment it becomes noisy or
+expensive. The checks here are the boring half of evaluation, and the boring
+half is the half that actually runs on every pull request.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from typing import Any
+
+#: Phrases that mean the assistant gave up rather than answered. These are
+#: deliberately broad and match both Chinese and English refusals.
+FALLBACK_PATTERNS: tuple[re.Pattern[str], ...] = tuple(
+    re.compile(pattern, re.IGNORECASE)
+    for pattern in (
+        r"作为一个\s*(AI|人工智能|语言模型)",
+        r"作为\s*(AI|人工智能|语言模型)",
+        r"我(无法|不能|没有办法)(回答|提供|满足|帮助|处理)",
+        r"抱歉[，,]?\s*我(无法|不能|不会)",
+        r"对不起[，,]?\s*我(无法|不能|不会)",
+        r"暂(时)?无法(回答|提供|处理)",
+        r"没有(相关|足够)的?(信息|资料)来(回答|说明)",
+        r"as an ai\b",
+        r"i (can'?t|cannot|am unable to) (help|assist|answer|provide)",
+        r"i'?m sorry,? but i (can'?t|cannot)",
+        r"i don'?t have (enough )?(information|access)",
+    )
+)
+
+SUPPORTED_CHECK_TYPES = (
+    "not_fallback",
+    "min_chars",
+    "max_chars",
+    "contains",
+    "not_contains",
+    "regex",
+    "json",
+)
+
+
+@dataclass
+class CheckResult:
+    """Outcome of a single check against a single output."""
+
+    type: str
+    passed: bool
+    detail: str = ""
+    spec: dict[str, Any] | None = None
+
+
+def detect_fallback(output: str) -> str | None:
+    """Return the first fallback phrase found, or ``None`` if the output is clean."""
+    for pattern in FALLBACK_PATTERNS:
+        match = pattern.search(output)
+        if match:
+            return match.group(0)
+    return None
+
+
+def _as_list(value: Any) -> list[str]:
+    if value is None:
+        return []
+    if isinstance(value, str):
+        return [value]
+    if isinstance(value, (list, tuple)):
+        return [str(item) for item in value]
+    return [str(value)]
+
+
+def run_check(check: dict[str, Any], output: str) -> CheckResult:
+    """Run one check spec against one output.
+
+    Unknown check types fail loudly rather than silently passing -- a test that
+    quietly does nothing is worse than no test at all.
+    """
+    check_type = str(check.get("type", "")).strip()
+    spec = dict(check)
+
+    if check_type == "not_fallback":
+        hit = detect_fallback(output)
+        if hit:
+            return CheckResult(check_type, False, f"matched fallback phrase {hit!r}", spec)
+        return CheckResult(check_type, True, "", spec)
+
+    if check_type == "min_chars":
+        limit = int(check.get("value", 0))
+        actual = len(output.strip())
+        if actual < limit:
+            return CheckResult(check_type, False, f"{actual} chars < {limit}", spec)
+        return CheckResult(check_type, True, f"{actual} chars", spec)
+
+    if check_type == "max_chars":
+        limit = int(check.get("value", 0))
+        actual = len(output.strip())
+        if actual > limit:
+            return CheckResult(check_type, False, f"{actual} chars > {limit}", spec)
+        return CheckResult(check_type, True, f"{actual} chars", spec)
+
+    if check_type == "contains":
+        needles = _as_list(check.get("value"))
+        missing = [needle for needle in needles if needle not in output]
+        if missing:
+            return CheckResult(check_type, False, f"missing {missing}", spec)
+        return CheckResult(check_type, True, "", spec)
+
+    if check_type == "not_contains":
+        needles = _as_list(check.get("value"))
+        present = [needle for needle in needles if needle in output]
+        if present:
+            return CheckResult(check_type, False, f"found forbidden {present}", spec)
+        return CheckResult(check_type, True, "", spec)
+
+    if check_type == "regex":
+        pattern = str(check.get("value", ""))
+        try:
+            compiled = re.compile(pattern)
+        except re.error as exc:
+            return CheckResult(check_type, False, f"invalid pattern: {exc}", spec)
+        if not compiled.search(output):
+            return CheckResult(check_type, False, f"no match for {pattern!r}", spec)
+        return CheckResult(check_type, True, "", spec)
+
+    if check_type == "json":
+        required = _as_list(check.get("required"))
+        try:
+            payload = json.loads(output)
+        except (json.JSONDecodeError, TypeError) as exc:
+            return CheckResult(check_type, False, f"invalid JSON: {exc}", spec)
+        if required:
+            if not isinstance(payload, dict):
+                return CheckResult(check_type, False, "expected a JSON object", spec)
+            missing = [key for key in required if key not in payload]
+            if missing:
+                return CheckResult(check_type, False, f"missing keys {missing}", spec)
+        return CheckResult(check_type, True, "", spec)
+
+    return CheckResult(check_type, False, f"unsupported check type {check_type!r}", spec)
+
+
+def run_checks(checks: list[dict[str, Any]], output: str) -> list[CheckResult]:
+    return [run_check(check, output) for check in checks]

trace2eval/cli.py

@@ -0,0 +1,307 @@
+"""Command line interface.
+
+Three commands, in the order you would actually use them:
+
+    trace2eval build   # log  -> case set
+    trace2eval run     # case set + outputs -> metrics
+    trace2eval check   # metrics vs metrics -> pass/fail
+
+``check`` exits non-zero on regression, which is the whole point: it is meant to
+be the last line of a CI job, not a report someone reads on a dashboard an hour
+later.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Any, Sequence
+
+from . import __version__
+from .report import build_report, render_metrics, render_violations
+from .runner import (
+    OutputFormatError,
+    RunMetrics,
+    compare_runs,
+    load_outputs,
+    run_cases,
+)
+from .schema import TraceFormatError, load_traces
+from .select import (
+    DEFAULT_DEDUP_THRESHOLD,
+    MatcherSpecError,
+    load_matcher,
+    select_cases,
+)
+
+EXIT_OK = 0
+EXIT_GATE_FAILED = 1
+EXIT_BAD_INPUT = 2
+
+
+def _write_text(path: Path, content: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    # newline="\n" disables platform newline translation. Without it, Windows
+    # writes CRLF and Linux writes LF, and the committed case set stops being
+    # byte-identical across platforms -- which is exactly what the CI job asserts.
+    path.write_text(content, encoding="utf-8", newline="\n")
+
+
+def _write_jsonl(path: Path, rows: Sequence[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8", newline="\n") as handle:
+        for row in rows:
+            handle.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+
+def _read_jsonl(path: Path) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    with path.open("r", encoding="utf-8") as handle:
+        for line in handle:
+            stripped = line.strip()
+            if stripped:
+                rows.append(json.loads(stripped))
+    return rows
+
+
+def cmd_build(args: argparse.Namespace) -> int:
+    try:
+        loaded = load_traces(args.traces)
+    except TraceFormatError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return EXIT_BAD_INPUT
+
+    if not loaded.traces:
+        print("error: no usable traces found in the input file", file=sys.stderr)
+        if loaded.skipped:
+            for line_no, reason in loaded.skipped[:5]:
+                print(f"  line {line_no}: {reason}", file=sys.stderr)
+        return EXIT_BAD_INPUT
+
+    matcher = None
+    if args.similarity:
+        try:
+            matcher = load_matcher(args.similarity)
+        except MatcherSpecError as exc:
+            print(f"error: --similarity {exc}", file=sys.stderr)
+            return EXIT_BAD_INPUT
+
+    result = select_cases(
+        loaded.traces,
+        max_cases=args.max_cases,
+        min_score=args.min_score,
+        dedup_threshold=args.dedup_threshold,
+        matcher=matcher,
+    )
+
+    out_dir = Path(args.out)
+    _write_jsonl(out_dir / "cases.jsonl", result.cases)
+    # as_posix() so the report is byte-identical on Windows and Linux, which is
+    # what lets CI diff the committed case set against a fresh build.
+    _write_text(out_dir / "report.md", build_report(result, Path(args.traces).as_posix()))
+
+    stats = result.stats()
+    print(f"read {stats['total_traces']} traces from {args.traces}")
+    if loaded.skipped:
+        print(f"  skipped {loaded.skipped_count} unreadable line(s)")
+    if matcher is not None:
+        print(
+            "  custom similarity matcher: the n-gram index is bypassed, "
+            "so this run compares against every cluster"
+        )
+    print(f"generated {stats['cases']} cases -> {out_dir / 'cases.jsonl'}")
+    print(f"  {stats['dropped_as_duplicate']} collapsed as near-duplicates")
+    print(f"  {stats['dropped_below_min_score']} below the minimum score")
+    print(f"  {stats['dropped_beyond_limit']} beyond the case limit")
+    if stats["cases_whose_reference_fails"]:
+        print(
+            f"  {stats['cases_whose_reference_fails']} trusted reference(s) fail their "
+            f"own checks -- look at these before committing the set"
+        )
+    if stats["cases_with_weak_checks"]:
+        print(
+            f"  {stats['cases_with_weak_checks']} case(s) carry checks that cannot "
+            f"detect the failure they came from"
+        )
+    print(f"report -> {out_dir / 'report.md'}")
+    return EXIT_OK
+
+
+def cmd_run(args: argparse.Namespace) -> int:
+    cases_path = Path(args.cases)
+    if not cases_path.exists():
+        print(f"error: case file not found: {cases_path}", file=sys.stderr)
+        return EXIT_BAD_INPUT
+
+    try:
+        cases = _read_jsonl(cases_path)
+        outputs = load_outputs(args.outputs)
+    except (OutputFormatError, json.JSONDecodeError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return EXIT_BAD_INPUT
+
+    metrics = run_cases(cases, outputs)
+
+    if args.out:
+        payload = {"version": __version__, "metrics": metrics.to_dict()}
+        _write_text(Path(args.out), json.dumps(payload, indent=2, ensure_ascii=False) + "\n")
+
+    print(render_metrics(metrics))
+    if args.out:
+        print(f"\nwritten -> {args.out}")
+
+    if metrics.failures and args.show_failures:
+        print("\nFailures:")
+        for failure in metrics.failures[:20]:
+            detail = ", ".join(failure["failed_checks"]) or failure["reason"]
+            print(f"  {failure['case_id']}: {detail}")
+        if len(metrics.failures) > 20:
+            print(f"  ... and {len(metrics.failures) - 20} more")
+
+    if args.min_pass_rate is not None and metrics.pass_rate < args.min_pass_rate:
+        print(
+            f"\nFAIL: pass rate {metrics.pass_rate:.2%} is below the required "
+            f"{args.min_pass_rate:.2%}",
+            file=sys.stderr,
+        )
+        return EXIT_GATE_FAILED
+
+    return EXIT_OK
+
+
+def cmd_check(args: argparse.Namespace) -> int:
+    try:
+        baseline_payload = json.loads(Path(args.baseline).read_text(encoding="utf-8"))
+        current_payload = json.loads(Path(args.current).read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        print(f"error: cannot read metrics file: {exc}", file=sys.stderr)
+        return EXIT_BAD_INPUT
+
+    baseline = RunMetrics.from_dict(baseline_payload.get("metrics", baseline_payload))
+    current = RunMetrics.from_dict(current_payload.get("metrics", current_payload))
+
+    print("baseline")
+    print(render_metrics(baseline))
+    print("\ncurrent")
+    print(render_metrics(current))
+
+    violations = compare_runs(baseline, current, tolerance_scale=args.tolerance_scale)
+
+    print("\nregressions")
+    print(render_violations(violations))
+
+    if violations:
+        print(f"\nFAIL: {len(violations)} regression(s) detected", file=sys.stderr)
+        return EXIT_GATE_FAILED
+
+    print("\nPASS: no regressions detected")
+    return EXIT_OK
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="trace2eval",
+        description=(
+            "Turn production LLM traces into a regression eval set. "
+            "Zero dependencies, no API keys, fully offline."
+        ),
+    )
+    parser.add_argument("--version", action="version", version=f"trace2eval {__version__}")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    build = subparsers.add_parser(
+        "build",
+        help="turn a JSONL trace log into a regression eval set",
+        description=(
+            "Score every trace, collapse near-duplicates, and emit the highest-value "
+            "subset as a case set."
+        ),
+    )
+    build.add_argument("traces", help="input JSONL trace log")
+    build.add_argument("-o", "--out", default="evalset", help="output directory (default: evalset)")
+    build.add_argument("--max-cases", type=int, default=50, help="cap on generated cases")
+    build.add_argument(
+        "--min-score",
+        type=float,
+        default=1.0,
+        help="minimum signal weight for a trace to become a case",
+    )
+    build.add_argument(
+        "--dedup-threshold",
+        type=float,
+        default=DEFAULT_DEDUP_THRESHOLD,
+        help=(
+            "overlap similarity above which two inputs are treated as the same "
+            f"question (default: {DEFAULT_DEDUP_THRESHOLD})"
+        ),
+    )
+    build.add_argument(
+        "--similarity",
+        default=None,
+        metavar="MODULE:FUNCTION",
+        help=(
+            "swap in your own matcher, e.g. 'my_embeddings:cosine'. Must take two "
+            "strings and return a score in [0, 1]. Disables the n-gram index, so "
+            "clustering gets slower -- see trace2eval.matchers."
+        ),
+    )
+    build.set_defaults(func=cmd_build)
+
+    run = subparsers.add_parser(
+        "run",
+        help="score a batch of outputs against a case set",
+        description="Apply the deterministic checks in a case set to a batch of outputs.",
+    )
+    run.add_argument("--cases", required=True, help="cases.jsonl produced by build")
+    run.add_argument("--outputs", required=True, help="JSONL of {id, output} records to score")
+    run.add_argument("-o", "--out", help="write metrics JSON to this path")
+    run.add_argument(
+        "--min-pass-rate",
+        type=float,
+        default=None,
+        help="exit non-zero if the pass rate falls below this value",
+    )
+    run.add_argument(
+        "--show-failures",
+        action="store_true",
+        help="list which cases failed and on which checks",
+    )
+    run.set_defaults(func=cmd_run)
+
+    check = subparsers.add_parser(
+        "check",
+        help="compare two metrics files and fail on regression",
+        description=(
+            "Compare a baseline metrics file against a current one. Exits 1 if any "
+            "tracked metric moved the wrong way beyond its tolerance."
+        ),
+    )
+    check.add_argument("--baseline", required=True, help="baseline metrics JSON")
+    check.add_argument("--current", required=True, help="current metrics JSON")
+    check.add_argument(
+        "--tolerance-scale",
+        type=float,
+        default=1.0,
+        help="multiply every tolerance; use <1 for a stricter gate",
+    )
+    check.set_defaults(func=cmd_check)
+
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(encoding="utf-8")  # type: ignore[union-attr]
+        except (AttributeError, OSError):
+            pass
+
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

trace2eval/matchers.py

@@ -0,0 +1,101 @@
+"""Alternative similarity matchers.
+
+The default matcher lives in :mod:`trace2eval.select` and works on character
+n-grams, because that is the only thing that handles Chinese and English through
+one code path without a dependency. Its blind spot is fixed and known: a
+paraphrase that shares no characters with the original scores zero.
+
+Anything with the signature ``(str, str) -> float`` returning a value in
+``[0, 1]`` can be swapped in:
+
+    trace2eval build traces.jsonl --similarity my_embeddings:cosine
+
+That is the intended answer to the paraphrase problem -- bring whatever model you
+already have rather than making this package depend on one. This module ships two
+dependency-free matchers: useful in their own right, and working examples of the
+hook if you want to write your own.
+
+Be aware of the cost when you swap: a custom matcher disables the n-gram inverted
+index used to shortlist clusters, because a lexical shortlist would cap exactly
+the recall you brought the custom matcher in for. Clustering then compares against
+every cluster, so it gets noticeably slower on a large log.
+"""
+
+from __future__ import annotations
+
+import re
+
+#: Latin/digit words. Split on everything else.
+_WORD_PATTERN = re.compile(r"[a-z0-9]+")
+
+#: CJK ideographs, treated as individual tokens because there are no word
+#: boundaries to split on.
+_CJK_PATTERN = re.compile(r"[\u4e00-\u9fff]")
+
+#: Below this many shared tokens the two inputs are unrelated as far as these
+#: matchers are concerned. Guards the same failure mode ``MIN_SHARED_SHINGLES``
+#: guards: two short inputs sharing one token scoring an overlap of 1.0.
+MIN_SHARED_TOKENS = 2
+
+
+def tokenise(text: str) -> set[str]:
+    """Split into a token set: words for Latin script, characters for CJK.
+
+    Mixed text gets the union, so an English error message embedded in a Chinese
+    question still contributes its words.
+    """
+    lowered = text.lower()
+    tokens = set(_WORD_PATTERN.findall(lowered))
+    tokens |= set(_CJK_PATTERN.findall(lowered))
+    return tokens
+
+
+def _overlap(left: set[str], right: set[str]) -> float:
+    if not left or not right:
+        return 0.0
+    shared = len(left & right)
+    if shared < MIN_SHARED_TOKENS:
+        return 0.0
+    return shared / min(len(left), len(right))
+
+
+def word_jaccard(left: str, right: str) -> float:
+    """Token-level overlap coefficient.
+
+    Trades the default matcher's precision for word-order robustness. Prefer it
+    when inputs are whole sentences in a whitespace-delimited language.
+
+    Two honest caveats:
+
+    - For Chinese it is *coarser* than the default. Character bigrams keep some
+      ordering information; single characters do not, so genuinely different
+      questions that share most of their characters score higher here. Measured
+      on the sample log: ``支持哪些登录方式`` vs ``支持哪些支付方式`` scores 0.571
+      under the default matcher and 0.857 under this one. If you use this on
+      Chinese traffic, raise ``--dedup-threshold``.
+    - It is a matcher for experimenting with the hook, not a recommendation. The
+      real upgrade for paraphrase recall is embeddings, which this package does
+      not bundle on purpose.
+    """
+    return _overlap(tokenise(left), tokenise(right))
+
+
+def char_bigram_jaccard(left: str, right: str) -> float:
+    """Plain Jaccard over character bigrams.
+
+    Here to be measured against, not used. ``tests/test_dedup.py`` asserts that it
+    ranks two different questions above two phrasings of the same question on the
+    sample log, which is the observation that ruled Jaccard out for the built-in
+    matcher. Useful if you want to reproduce that measurement on your own data.
+    """
+    from .select import SHINGLE_SIZE, jaccard, normalise
+
+    def grams(text: str) -> set[str]:
+        folded = normalise(text)
+        if not folded:
+            return set()
+        if len(folded) <= SHINGLE_SIZE:
+            return {folded}
+        return {folded[i : i + SHINGLE_SIZE] for i in range(len(folded) - SHINGLE_SIZE + 1)}
+
+    return jaccard(grams(left), grams(right))