trace2eval-cli 0.2.1__py3-none-any.whl

trace2eval/report.py

@@ -0,0 +1,221 @@
+"""Markdown rendering for the generated case set and for regression output."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .runner import RunMetrics, Violation
+from .select import SelectionResult
+
+
+def _cell(value: Any, limit: int = 60) -> str:
+    """Make a value safe to drop into a Markdown table cell."""
+    text = str(value).replace("|", "\\|").replace("\n", " ").strip()
+    if len(text) > limit:
+        text = text[: limit - 1] + "…"
+    return text or "—"
+
+
+def _number(value: Any, digits: int = 4) -> str:
+    if value is None:
+        return "—"
+    if isinstance(value, float):
+        if value == 0:
+            return "0"
+        if abs(value) < 0.001:
+            return f"{value:.8f}".rstrip("0")
+        return f"{value:.{digits}f}"
+    return str(value)
+
+
+def build_report(result: SelectionResult, source_name: str) -> str:
+    """Human-readable account of what was selected and, more importantly, why."""
+    stats = result.stats()
+    context = result.context
+    lines: list[str] = []
+
+    lines.append("# trace2eval build report")
+    lines.append("")
+    lines.append(f"Source log: `{source_name}`")
+    lines.append("")
+
+    lines.append("## Summary")
+    lines.append("")
+    lines.append("| Metric | Value |")
+    lines.append("| --- | --- |")
+    lines.append(f"| Traces read | {stats['total_traces']} |")
+    lines.append(f"| Distinct questions | {stats['distinct_questions']} |")
+    lines.append(f"| Questions with no signal | {stats['questions_without_signals']} |")
+    lines.append(f"| Cases generated | {stats['cases']} |")
+    lines.append(f"| Log lines collapsed into a case | {stats['dropped_as_duplicate']} |")
+    lines.append(f"| Questions below the minimum score | {stats['dropped_below_min_score']} |")
+    lines.append(f"| Questions beyond the case limit | {stats['dropped_beyond_limit']} |")
+    lines.append(
+        f"| Cases whose checks cannot catch their own failure "
+        f"| {stats['cases_with_weak_checks']} |"
+    )
+    lines.append(
+        f"| Trusted references failing their own checks "
+        f"| {stats['cases_whose_reference_fails']} |"
+    )
+    lines.append("")
+
+    lines.append("## Log-wide baselines")
+    lines.append("")
+    lines.append("| Statistic | Value |")
+    lines.append("| --- | --- |")
+    lines.append(f"| p95 latency | {_number(context.p95_latency_ms, 2)} ms |")
+    lines.append(f"| p95 cost | ${_number(context.p95_cost_usd, 8)} |")
+    lines.append(f"| Median output length | {_number(context.median_output_chars, 1)} chars |")
+    lines.append("")
+
+    failures = sum(1 for case in result.cases if not case["reference_is_trusted"])
+    lines.append("## Case mix")
+    lines.append("")
+    lines.append(
+        f"- **{failures}** regression seeds (the reference output is the output that "
+        f"went wrong — the case exists so it never ships twice)"
+    )
+    lines.append(
+        f"- **{len(result.cases) - failures}** quality anchors (clean trace, so an "
+        f"expected shape was inferred from it)"
+    )
+    lines.append("")
+
+    lines.append("## Cases")
+    lines.append("")
+    lines.append("| Case | Score | In log | Trusted | Shape from | Self-check | Signals | Input |")
+    lines.append("| --- | --- | --- | --- | --- | --- | --- | --- |")
+    for case in result.cases:
+        signal_names = ", ".join(signal["name"] for signal in case["signals"])
+        lines.append(
+            "| {id} | {score} | {occurs} | {trusted} | {shape} | {check} | {signals} | {input} |".format(
+                id=_cell(case["id"], 12),
+                score=_cell(case["score"], 8),
+                occurs=_cell(case["occurrences_in_log"], 8),
+                trusted="yes" if case["reference_is_trusted"] else "no",
+                shape=_cell(case["shape_source"], 30),
+                check=_cell(case["self_check"]["verdict"], 30),
+                signals=_cell(signal_names, 42),
+                input=_cell(case["input"], 40),
+            )
+        )
+    lines.append("")
+
+    flagged = result.cases_whose_reference_fails + result.cases_with_weak_checks
+    if flagged:
+        lines.append("## Cases that need a human eye")
+        lines.append("")
+        lines.append(
+            "Generated cases are a proposal. These are the ones where the proposal "
+            "is weakest, in order of how much they should worry you."
+        )
+        lines.append("")
+        for case in result.cases_whose_reference_fails:
+            lines.append(
+                f"- **{case['id']}** — a trusted reference does not satisfy its own "
+                f"checks (`{', '.join(case['self_check']['failed_checks'])}`). Either "
+                f"the reference is wrong or the checks are."
+            )
+        for case in result.cases_with_weak_checks:
+            if case["failure_kind"] == "behaviour":
+                lines.append(
+                    f"- {case['id']} — the failure here was behavioural "
+                    f"(`{', '.join(signal['name'] for signal in case['signals'])}`): the "
+                    f"call itself produced a perfectly acceptable answer, and what went "
+                    f"wrong happened around it. No check on the output text can "
+                    f"reproduce that. Keep the case as a pinned input, but it needs a "
+                    f"labelled expected answer before it can gate anything."
+                )
+            else:
+                lines.append(
+                    f"- {case['id']} — the failure was visible in the output "
+                    f"(`{', '.join(signal['name'] for signal in case['signals'])}`), yet "
+                    f"the generated checks pass on it. That is a gap in the checks, not "
+                    f"a limit of the approach -- worth investigating."
+                )
+        lines.append("")
+
+    if result.cases:
+        lines.append("## Why the top case was chosen")
+        lines.append("")
+        top = result.cases[0]
+        lines.append(f"`{top['id']}` scored **{top['score']}**.")
+        lines.append("")
+        for signal in top["signals"]:
+            lines.append(f"- `{signal['name']}` (+{signal['weight']}) — {signal['detail']}")
+        lines.append("")
+        lines.append(f"The expected shape came from {top['shape_source']}.")
+        lines.append("")
+        check = top["self_check"]
+        if top["reference_is_trusted"]:
+            lines.append(
+                "Self-check: this is a trusted reference, and it "
+                + ("satisfies its own checks." if check["passed"] else "does NOT satisfy them.")
+            )
+        else:
+            if not check["passed"]:
+                detail = (
+                    f"fail on it — they do, on `{', '.join(check['failed_checks'])}`. "
+                    "The case reproduces the failure it came from."
+                )
+            else:
+                detail = (
+                    "fail on it — they do not. The case cannot detect the failure it "
+                    "came from, which is why it appears in the list above."
+                )
+            lines.append(
+                "Self-check: this is a failure seed, so the checks are *supposed* to "
+                + detail
+            )
+        lines.append("")
+        for note in top["notes"]:
+            lines.append(f"> {note}")
+        lines.append("")
+
+    lines.append("---")
+    lines.append("")
+    lines.append(
+        "Generated by [trace2eval](https://github.com/rfioly/trace2eval). "
+        "Review this file before committing the case set — the selection is a "
+        "proposal, not a verdict."
+    )
+    lines.append("")
+    return "\n".join(lines)
+
+
+def render_metrics(metrics: RunMetrics) -> str:
+    lines: list[str] = []
+    lines.append("| Metric | Value |")
+    lines.append("| --- | --- |")
+    lines.append(f"| Cases | {metrics.total} |")
+    lines.append(f"| Passed | {metrics.passed} |")
+    lines.append(f"| Failed | {metrics.failed} |")
+    if metrics.missing:
+        lines.append(f"| Missing outputs | {metrics.missing} |")
+    lines.append(f"| Pass rate | {metrics.pass_rate:.2%} |")
+    lines.append(f"| Fallback rate | {metrics.fallback_rate:.2%} |")
+    if metrics.format_compliance_rate is not None:
+        lines.append(f"| Format compliance | {metrics.format_compliance_rate:.2%} |")
+    lines.append(f"| Avg latency | {_number(metrics.avg_latency_ms, 2)} ms |")
+    lines.append(f"| p95 latency | {_number(metrics.p95_latency_ms, 2)} ms |")
+    lines.append(f"| Avg cost | ${_number(metrics.avg_cost_usd, 8)} |")
+    return "\n".join(lines)
+
+
+def render_violations(violations: list[Violation]) -> str:
+    if not violations:
+        return "No regressions detected."
+    lines: list[str] = []
+    lines.append("| Metric | Baseline | Current | Rule |")
+    lines.append("| --- | --- | --- | --- |")
+    for violation in violations:
+        lines.append(
+            "| {metric} | {baseline} | {current} | {rule} |".format(
+                metric=_cell(violation.metric, 24),
+                baseline=_cell(_number(violation.baseline), 16),
+                current=_cell(_number(violation.current), 16),
+                rule=_cell(violation.rule, 40),
+            )
+        )
+    return "\n".join(lines)

trace2eval/runner.py

@@ -0,0 +1,251 @@
+"""Running a case set and comparing two runs.
+
+``run`` turns a case set plus a batch of outputs into a metrics document.
+``check`` compares two metrics documents and reports regressions.
+
+The split matters: metrics are a plain JSON file that can be committed, diffed,
+and read by any CI system. A gate that only exists inside a dashboard is not a
+gate.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any, Iterable
+
+from .checks import CheckResult, run_checks
+from .signals import percentile
+
+
+class OutputFormatError(ValueError):
+    """Raised when an outputs file cannot be read at all."""
+
+
+@dataclass
+class CaseOutcome:
+    case_id: str
+    passed: bool
+    results: list[CheckResult] = field(default_factory=list)
+    output_chars: int = 0
+    latency_ms: float | None = None
+    cost_usd: float | None = None
+
+    def failed_check_types(self) -> list[str]:
+        return [result.type for result in self.results if not result.passed]
+
+
+def load_outputs(path: str | Path) -> dict[str, dict[str, Any]]:
+    """Read a JSONL file of ``{"id": ..., "output": ...}`` records."""
+    output_path = Path(path)
+    if not output_path.exists():
+        raise OutputFormatError(f"outputs file not found: {output_path}")
+
+    records: dict[str, dict[str, Any]] = {}
+    with output_path.open("r", encoding="utf-8") as handle:
+        for line_no, line in enumerate(handle, start=1):
+            stripped = line.strip()
+            if not stripped:
+                continue
+            try:
+                data = json.loads(stripped)
+            except json.JSONDecodeError:
+                continue
+            if not isinstance(data, dict):
+                continue
+            record_id = data.get("id") or data.get("trace_id")
+            if record_id is None:
+                record_id = f"trace-{line_no:05d}"
+            records[str(record_id)] = data
+    return records
+
+
+@dataclass
+class RunMetrics:
+    total: int = 0
+    passed: int = 0
+    failed: int = 0
+    missing: int = 0
+    pass_rate: float = 0.0
+    fallback_count: int = 0
+    fallback_rate: float = 0.0
+    json_cases: int = 0
+    json_passed: int = 0
+    format_compliance_rate: float | None = None
+    avg_latency_ms: float | None = None
+    p95_latency_ms: float | None = None
+    avg_cost_usd: float | None = None
+    total_cost_usd: float | None = None
+    avg_output_chars: float | None = None
+    failures: list[dict[str, Any]] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> RunMetrics:
+        known = {f for f in cls.__dataclass_fields__}
+        return cls(**{key: value for key, value in data.items() if key in known})
+
+
+def _round(value: float | None, digits: int = 4) -> float | None:
+    return None if value is None else round(value, digits)
+
+
+def run_cases(
+    cases: Iterable[dict[str, Any]],
+    outputs: dict[str, dict[str, Any]],
+) -> RunMetrics:
+    """Score every case against the matching output."""
+    case_list = list(cases)
+    metrics = RunMetrics(total=len(case_list))
+    latency_values: list[float] = []
+    cost_values: list[float] = []
+    char_values: list[int] = []
+
+    for case in case_list:
+        case_id = str(case.get("id", ""))
+        source_id = str(case.get("source_trace_id", ""))
+        record = outputs.get(case_id) or outputs.get(source_id)
+
+        if record is None:
+            metrics.failed += 1
+            metrics.missing += 1
+            metrics.failures.append(
+                {
+                    "case_id": case_id,
+                    "reason": "missing_output",
+                    "failed_checks": [],
+                }
+            )
+            continue
+
+        output = record.get("output") or record.get("response") or ""
+        output = str(output)
+        checks = case.get("checks", [])
+        results = run_checks(checks, output)
+        passed = all(result.passed for result in results)
+
+        outcome = CaseOutcome(case_id=case_id, passed=passed, results=results)
+        outcome.output_chars = len(output.strip())
+
+        latency = record.get("latency_ms")
+        if isinstance(latency, (int, float)):
+            outcome.latency_ms = float(latency)
+            latency_values.append(float(latency))
+        cost = record.get("cost_usd")
+        if isinstance(cost, (int, float)):
+            outcome.cost_usd = float(cost)
+            cost_values.append(float(cost))
+        char_values.append(outcome.output_chars)
+
+        if passed:
+            metrics.passed += 1
+        else:
+            metrics.failed += 1
+            metrics.failures.append(
+                {
+                    "case_id": case_id,
+                    "reason": "check_failed",
+                    "failed_checks": outcome.failed_check_types(),
+                }
+            )
+
+        if any(check.get("type") == "not_fallback" for check in checks):
+            if not passed and "not_fallback" in outcome.failed_check_types():
+                metrics.fallback_count += 1
+
+        if any(check.get("type") == "json" for check in checks):
+            metrics.json_cases += 1
+            if passed:
+                metrics.json_passed += 1
+
+    denominator = metrics.total or 1
+    metrics.pass_rate = _round(metrics.passed / denominator, 4) or 0.0
+    metrics.fallback_rate = _round(metrics.fallback_count / denominator, 4) or 0.0
+
+    if metrics.json_cases:
+        metrics.format_compliance_rate = _round(
+            metrics.json_passed / metrics.json_cases, 4
+        )
+
+    if latency_values:
+        metrics.avg_latency_ms = _round(sum(latency_values) / len(latency_values), 2)
+        metrics.p95_latency_ms = _round(percentile(latency_values, 0.95), 2)
+    if cost_values:
+        metrics.avg_cost_usd = _round(sum(cost_values) / len(cost_values), 8)
+        metrics.total_cost_usd = _round(sum(cost_values), 6)
+    if char_values:
+        metrics.avg_output_chars = _round(sum(char_values) / len(char_values), 1)
+
+    return metrics
+
+
+@dataclass
+class Violation:
+    metric: str
+    baseline: float | None
+    current: float | None
+    rule: str
+
+
+#: (metric, direction, tolerance). "higher_is_better" fails when it drops,
+#: "lower_is_better" fails when it rises.
+REGRESSION_RULES: tuple[tuple[str, str, float], ...] = (
+    ("pass_rate", "higher_is_better", 0.02),
+    ("format_compliance_rate", "higher_is_better", 0.02),
+    ("fallback_rate", "lower_is_better", 0.02),
+    ("p95_latency_ms", "lower_is_better", 0.20),
+    ("avg_cost_usd", "lower_is_better", 0.20),
+)
+
+#: Some metrics tolerate movement, so the baseline is nudged by this fraction
+#: before comparing. Pass rate is judged on absolute points instead.
+_RELATIVE_METRICS = {"p95_latency_ms", "avg_cost_usd"}
+
+
+def compare_runs(
+    baseline: RunMetrics,
+    current: RunMetrics,
+    tolerance_scale: float = 1.0,
+) -> list[Violation]:
+    """Return every metric that regressed beyond its tolerance."""
+    violations: list[Violation] = []
+    baseline_data = baseline.to_dict()
+    current_data = current.to_dict()
+
+    for metric, direction, tolerance in REGRESSION_RULES:
+        base = baseline_data.get(metric)
+        now = current_data.get(metric)
+        if base is None or now is None:
+            continue
+
+        tolerance = tolerance * tolerance_scale
+        if metric in _RELATIVE_METRICS:
+            allowed = base * tolerance
+        else:
+            allowed = tolerance
+
+        if direction == "higher_is_better":
+            if now < base - allowed:
+                violations.append(
+                    Violation(
+                        metric,
+                        base,
+                        now,
+                        f"dropped by {base - now:.4f} (allowed {allowed:.4f})",
+                    )
+                )
+        else:
+            if now > base + allowed:
+                violations.append(
+                    Violation(
+                        metric,
+                        base,
+                        now,
+                        f"rose by {now - base:.4f} (allowed {allowed:.4f})",
+                    )
+                )
+
+    return violations

trace2eval/schema.py

@@ -0,0 +1,233 @@
+"""Trace loading and normalisation.
+
+The reader is deliberately forgiving: real production logs are messy and field
+names differ between frameworks. We accept a handful of common aliases for the
+input/output pair, and we skip malformed lines instead of aborting the whole
+run -- a single bad line in a 200k-line log must not cost you the batch.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+INPUT_ALIASES = ("input", "prompt", "question", "query", "user_message")
+OUTPUT_ALIASES = ("output", "response", "completion", "answer", "assistant_message")
+ID_ALIASES = ("id", "trace_id", "traceId", "request_id", "requestId")
+
+_NEGATIVE_FEEDBACK = {
+    "negative",
+    "neg",
+    "bad",
+    "down",
+    "thumbs_down",
+    "thumbsdown",
+    "差评",
+    "不满意",
+    "无帮助",
+}
+_POSITIVE_FEEDBACK = {
+    "positive",
+    "pos",
+    "good",
+    "up",
+    "thumbs_up",
+    "thumbsup",
+    "好评",
+    "满意",
+    "有帮助",
+}
+
+
+class TraceFormatError(ValueError):
+    """Raised when the input file cannot be read at all."""
+
+
+@dataclass
+class Trace:
+    """One recorded call.
+
+    Only ``input`` and ``output`` are required. Everything else is optional and
+    simply widens the set of signals we can compute for this row.
+    """
+
+    id: str
+    input: str
+    output: str
+    latency_ms: float | None = None
+    cost_usd: float | None = None
+    feedback: str | None = None
+    retried: bool = False
+    expect: dict[str, Any] | None = None
+    raw: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def output_chars(self) -> int:
+        return len(self.output.strip())
+
+    @property
+    def is_negative(self) -> bool:
+        return self.feedback == "negative"
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "input": self.input,
+            "output": self.output,
+            "latency_ms": self.latency_ms,
+            "cost_usd": self.cost_usd,
+            "feedback": self.feedback,
+            "retried": self.retried,
+            "expect": self.expect,
+        }
+
+
+@dataclass
+class LoadResult:
+    traces: list[Trace] = field(default_factory=list)
+    skipped: list[tuple[int, str]] = field(default_factory=list)
+
+    @property
+    def skipped_count(self) -> int:
+        return len(self.skipped)
+
+
+def _pick(data: dict[str, Any], keys: tuple[str, ...], allow_empty: bool = False) -> Any:
+    """Return the first alias that is present and usable.
+
+    ``allow_empty`` exists because an empty string means different things for the
+    two sides of a call. An empty *input* is an unusable row -- there was nothing
+    to answer. An empty *output* is a perfectly readable row and one of the most
+    valuable ones in the log: it means the model returned nothing at all.
+    """
+    for key in keys:
+        if key in data:
+            value = data[key]
+            if value is None:
+                continue
+            if isinstance(value, str) and not value.strip() and not allow_empty:
+                continue
+            return value
+    return None
+
+
+def _as_float(value: Any) -> float | None:
+    if value is None or isinstance(value, bool):
+        return None
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _as_bool(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, (int, float)):
+        return value > 0
+    if isinstance(value, str):
+        return value.strip().lower() in {"1", "true", "yes", "y"}
+    return False
+
+
+def normalise_feedback(value: Any) -> str | None:
+    """Collapse the many spellings of a thumbs-up/down into one of two values."""
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return "positive" if value else "negative"
+    text = str(value).strip().lower()
+    if not text or text in {"none", "null", "unknown", "-"}:
+        return None
+    if text in _NEGATIVE_FEEDBACK:
+        return "negative"
+    if text in _POSITIVE_FEEDBACK:
+        return "positive"
+    # Numeric ratings: 1-2 is a thumbs-down, 4-5 is a thumbs-up.
+    try:
+        score = float(text)
+    except ValueError:
+        return None
+    if score <= 2:
+        return "negative"
+    if score >= 4:
+        return "positive"
+    return None
+
+
+def parse_trace(data: dict[str, Any], fallback_id: str) -> Trace | None:
+    """Build a :class:`Trace` from a raw log record, or ``None`` if unusable."""
+    raw_input = _pick(data, INPUT_ALIASES)
+    raw_output = _pick(data, OUTPUT_ALIASES, allow_empty=True)
+    if raw_input is None or raw_output is None:
+        return None
+
+    trace_id = _pick(data, ID_ALIASES) or fallback_id
+    retried = _as_bool(_pick(data, ("retried", "is_retry", "was_retried")))
+    retry_count = _as_float(_pick(data, ("retry_count", "retries")))
+    if retry_count:
+        retried = True
+
+    latency = _as_float(
+        _pick(data, ("latency_ms", "latency", "duration_ms", "elapsed_ms"))
+    )
+    if latency is None:
+        maybe_seconds = _as_float(_pick(data, ("latency_s", "duration_s")))
+        if maybe_seconds is not None:
+            latency = maybe_seconds * 1000.0
+
+    cost = _as_float(_pick(data, ("cost_usd", "cost", "total_cost", "price_usd")))
+
+    expect = data.get("expect") or data.get("assert") or data.get("checks")
+    if not isinstance(expect, dict):
+        expect = None
+
+    return Trace(
+        id=str(trace_id),
+        input=str(raw_input),
+        output=str(raw_output),
+        latency_ms=latency,
+        cost_usd=cost,
+        feedback=normalise_feedback(
+            _pick(data, ("feedback", "rating", "score", "user_feedback", "thumbs"))
+        ),
+        retried=retried,
+        expect=expect,
+        raw=data,
+    )
+
+
+def load_traces(path: str | Path) -> LoadResult:
+    """Read a JSONL trace log.
+
+    Blank lines are ignored. A line that is not valid JSON, or that lacks an
+    input/output pair, is recorded in ``skipped`` and everything else still
+    loads.
+    """
+    log_path = Path(path)
+    if not log_path.exists():
+        raise TraceFormatError(f"trace file not found: {log_path}")
+
+    result = LoadResult()
+    with log_path.open("r", encoding="utf-8") as handle:
+        for line_no, line in enumerate(handle, start=1):
+            stripped = line.strip()
+            if not stripped:
+                continue
+            try:
+                data = json.loads(stripped)
+            except json.JSONDecodeError as exc:
+                result.skipped.append((line_no, f"invalid JSON: {exc.msg}"))
+                continue
+            if not isinstance(data, dict):
+                result.skipped.append((line_no, "not a JSON object"))
+                continue
+            trace = parse_trace(data, fallback_id=f"trace-{line_no:05d}")
+            if trace is None:
+                result.skipped.append((line_no, "missing input/output field"))
+                continue
+            result.traces.append(trace)
+
+    return result