trace2eval-cli 0.2.1__py3-none-any.whl

trace2eval/signals.py

@@ -0,0 +1,207 @@
+"""Signals: deciding which traces are worth turning into test cases.
+
+A production log is mostly boring. If you dump ten thousand calls into a test
+suite you get a suite that is slow, redundant, and ignored. What you want is the
+handful of calls that carry information -- the ones where something already went
+wrong, or where the shape of the answer matters.
+
+Each signal below is a cheap, deterministic observation about a single row plus
+a little global context (percentiles across the whole log). Signals carry
+weights, and weights add up into a score. The score decides what gets promoted.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import statistics
+from dataclasses import dataclass, field
+from typing import Any, Iterable
+
+from .checks import detect_fallback
+from .schema import Trace
+
+#: Default weights. Tuned so that a single strong signal outranks several weak
+#: ones: an explicit thumbs-down should always beat "this call was a bit slow".
+DEFAULT_WEIGHTS: dict[str, float] = {
+    "negative_feedback": 3.0,
+    "user_retried": 2.5,
+    "empty_output": 3.0,
+    "expected_shape_violated": 3.0,
+    "fallback_phrase": 2.0,
+    "explicit_expectation": 2.0,
+    "output_much_shorter": 1.5,
+    "output_much_longer": 1.0,
+    "slow_response": 1.0,
+    "expensive_call": 1.0,
+}
+
+#: An output shorter than this is treated as suspicious on its own.
+MIN_PLAUSIBLE_CHARS = 12
+
+#: A response is "much shorter/longer" past these ratios against the log median.
+SHORT_RATIO = 0.5
+LONG_RATIO = 3.0
+
+# Backwards-compatible private aliases; the underscore names are historical.
+_MIN_PLAUSIBLE_CHARS = MIN_PLAUSIBLE_CHARS
+_SHORT_RATIO = SHORT_RATIO
+_LONG_RATIO = LONG_RATIO
+
+
+def is_suspiciously_short(chars: int, median_output_chars: float) -> bool:
+    """Whether a response this long counts as too short for this log."""
+    if median_output_chars <= 0:
+        return False
+    return (
+        chars < median_output_chars * SHORT_RATIO
+        and chars < MIN_PLAUSIBLE_CHARS * 2
+    )
+
+
+def suspiciously_short_boundary(median_output_chars: float) -> int | None:
+    """The length a response had to fall below to be flagged as too short.
+
+    Exposed because the generated *checks* need the same number the *signal* used.
+    Deriving both from one function is the only way to stop them drifting apart:
+    if the threshold moved and the checks did not follow, a case built from a
+    length failure would stop reproducing its own failure, silently.
+    """
+    if median_output_chars <= 0:
+        return None
+    return int(math.ceil(min(median_output_chars * SHORT_RATIO, MIN_PLAUSIBLE_CHARS * 2)))
+
+
+@dataclass
+class Signal:
+    name: str
+    weight: float
+    detail: str
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"name": self.name, "weight": self.weight, "detail": self.detail}
+
+
+@dataclass
+class TraceContext:
+    """Log-wide statistics that individual signals are judged against."""
+
+    p95_latency_ms: float | None = None
+    p95_cost_usd: float | None = None
+    median_output_chars: float = 0.0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "p95_latency_ms": self.p95_latency_ms,
+            "p95_cost_usd": self.p95_cost_usd,
+            "median_output_chars": round(self.median_output_chars, 1),
+        }
+
+
+def percentile(values: list[float], fraction: float) -> float | None:
+    """Linear-interpolation percentile. Avoids depending on numpy for one number."""
+    clean = sorted(value for value in values if value is not None)
+    if not clean:
+        return None
+    if len(clean) == 1:
+        return clean[0]
+    position = fraction * (len(clean) - 1)
+    lower = int(position)
+    upper = min(lower + 1, len(clean) - 1)
+    weight = position - lower
+    return clean[lower] * (1 - weight) + clean[upper] * weight
+
+
+def build_context(traces: Iterable[Trace]) -> TraceContext:
+    trace_list = list(traces)
+    latencies = [t.latency_ms for t in trace_list if t.latency_ms is not None]
+    costs = [t.cost_usd for t in trace_list if t.cost_usd is not None]
+    char_counts = [t.output_chars for t in trace_list if t.output_chars > 0]
+    return TraceContext(
+        p95_latency_ms=percentile(latencies, 0.95),
+        p95_cost_usd=percentile(costs, 0.95),
+        median_output_chars=statistics.median(char_counts) if char_counts else 0.0,
+    )
+
+
+def _looks_like_json(text: str) -> bool:
+    stripped = text.strip()
+    if not stripped or stripped[0] not in "[{":
+        return False
+    try:
+        json.loads(stripped)
+    except (json.JSONDecodeError, TypeError):
+        return False
+    return True
+
+
+def compute_signals(
+    trace: Trace,
+    context: TraceContext,
+    weights: dict[str, float] | None = None,
+) -> list[Signal]:
+    """Collect every signal that fires for this trace."""
+    active = {**DEFAULT_WEIGHTS, **(weights or {})}
+    signals: list[Signal] = []
+
+    def add(name: str, detail: str) -> None:
+        weight = active.get(name, 0.0)
+        if weight > 0:
+            signals.append(Signal(name=name, weight=weight, detail=detail))
+
+    if trace.is_negative:
+        add("negative_feedback", "user gave negative feedback")
+
+    if trace.retried:
+        add("user_retried", "the same request was retried")
+
+    chars = trace.output_chars
+    if chars == 0:
+        add("empty_output", "model returned an empty response")
+    else:
+        fallback = detect_fallback(trace.output)
+        if fallback:
+            add("fallback_phrase", f"output contains {fallback!r}")
+
+        median = context.median_output_chars
+        if is_suspiciously_short(chars, median):
+            add(
+                "output_much_shorter",
+                f"{chars} chars vs log median {median:.0f}",
+            )
+        elif median > 0 and chars > median * LONG_RATIO:
+            add(
+                "output_much_longer",
+                f"{chars} chars vs log median {median:.0f}",
+            )
+
+    if trace.expect:
+        add("explicit_expectation", "trace already carried an expectation block")
+        if trace.expect.get("json") and not _looks_like_json(trace.output):
+            add("expected_shape_violated", "expected JSON output, got something else")
+
+    if (
+        context.p95_latency_ms is not None
+        and trace.latency_ms is not None
+        and trace.latency_ms > context.p95_latency_ms
+    ):
+        add(
+            "slow_response",
+            f"{trace.latency_ms:.0f}ms above p95 {context.p95_latency_ms:.0f}ms",
+        )
+
+    if (
+        context.p95_cost_usd is not None
+        and trace.cost_usd is not None
+        and trace.cost_usd > context.p95_cost_usd
+    ):
+        add(
+            "expensive_call",
+            f"${trace.cost_usd:.6f} above p95 ${context.p95_cost_usd:.6f}",
+        )
+
+    return signals
+
+
+def score(signals: list[Signal]) -> float:
+    return round(sum(signal.weight for signal in signals), 3)

trace2eval_cli-0.2.1.dist-info/METADATA

@@ -0,0 +1,227 @@
+Metadata-Version: 2.4
+Name: trace2eval-cli
+Version: 0.2.1
+Summary: Turn production LLM traces into a regression eval set. Zero dependencies.
+Author: rfioly
+License: MIT
+Project-URL: Homepage, https://github.com/rfioly/trace2eval
+Project-URL: Source, https://github.com/rfioly/trace2eval
+Project-URL: Issues, https://github.com/rfioly/trace2eval/issues
+Keywords: llm,evaluation,evals,observability,regression-testing,trace,ci
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Dynamic: license-file
+
+[English](README.en.md) | **简体中文**
+
+# trace2eval
+
+**把生产环境的 LLM 调用日志，变成一套回归测试集。**
+
+它是 Pytest 那个路子，区别在于用例不用手写——从你已经发生的线上调用里挖出来。
+
+零依赖、不用 API key、不调模型。同样的日志进，同样的用例出。
+
+[![CI](https://github.com/rfioly/trace2eval/actions/workflows/ci.yml/badge.svg)](https://github.com/rfioly/trace2eval/actions/workflows/ci.yml)
+[![Python](https://img.shields.io/badge/python-3.10%2B-blue)](https://www.python.org/)
+[![Dependencies](https://img.shields.io/badge/dependencies-0-brightgreen)](#)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
+
+---
+
+## 快速开始
+
+```bash
+pip install -e .
+
+# 1. 把日志变成用例集
+trace2eval build traces.jsonl -o evalset/
+
+# 2. 拿用例集给一批输出打分
+trace2eval run --cases evalset/cases.jsonl --outputs outputs.jsonl -o runs/current.json
+
+# 3. 有回归就失败（退出码 1），可直接挂 CI
+trace2eval check --baseline runs/baseline.json --current runs/current.json
+```
+
+需要 Python 3.10+。
+
+---
+
+## 三个命令
+
+| 命令 | 做什么 |
+| --- | --- |
+| `build` | JSONL 调用日志 → 用例集（`cases.jsonl` + `report.md`） |
+| `run` | 用例集 + 一批输出 → `metrics.json` |
+| `check` | 两份指标对比 → 通过 / 失败 |
+
+---
+
+## 特性
+
+- **零依赖，不用 API key。** 纯标准库，不调任何模型，完全离线可跑。
+- **确定性。** 同一份日志产出同一套用例，逐字节一致——这条在 CI 里有断言。
+- **一个问题一个用例。** 被问了 200 次的问题只变成一个用例，并且老实记录它代表 200 次调用。
+- **形状从干净答案学，不从失败那条学。** 失败种子的长度下限来自同问题的干净答案中位数。
+- **每个用例自检。** 报告里会标明哪些用例的检查**抓不住**它当初为什么坏，不混进"已覆盖"里。
+- **相似度可换。** `--similarity module:function` 换成你自己的实现，包本身不引依赖。
+- **天生适合 CI。** `check` 检出回归返回非零退出码。
+
+---
+
+## 跑一遍看看
+
+```console
+$ trace2eval build examples/sample_traces.jsonl -o evalset
+read 42 traces from examples/sample_traces.jsonl
+generated 16 cases -> evalset/cases.jsonl
+  8 collapsed as near-duplicates
+  3 case(s) carry checks that cannot detect the failure they came from
+report -> evalset/report.md
+```
+
+42 次调用 → 16 个用例。每个用例都记了为什么被挑中、形状从哪来、自检结论：
+
+```markdown
+| Case     | Score | In log | Shape from                     | Self-check             | Input                    |
+| -------- | ----- | ------ | ------------------------------ | ---------------------- | ------------------------ |
+| case-001 | 8.0   | 1      | 日志的短答线（弱参考）           | ok                     | 订单一直显示处理中，已经三天了 |
+| case-007 | 4.5   | 6      | 同一问题的 5 个干净答案          | ok                     | 你们的退款政策是什么？       |
+| case-011 | 2.5   | 4      | 同一问题的 3 个干净答案          | failure_not_reproduced | 修改手机号                |
+```
+
+再看门禁。改一次提示词，修好一处、弄坏五处：
+
+```console
+$ trace2eval check --baseline runs/baseline.json --current runs/current.json
+| Metric                  | Baseline | Current | Rule                                |
+| ----------------------- | -------- | ------- | ----------------------------------- |
+| pass_rate               | 1.0000   | 0.6875  | dropped by 0.3125 (allowed 0.0200)  |
+| format_compliance_rate  | 1.0000   | 0.5000  | dropped by 0.5000 (allowed 0.0200)  |
+| fallback_rate           | 0        | 0.0625  | rose by 0.0625 (allowed 0.0200)     |
+
+FAIL: 3 regression(s) detected
+$ echo $?
+1
+```
+
+五个失败横跨三种失败模式：
+
+```
+case-001: min_chars                 <- 答案被截断
+case-002: not_fallback, min_chars   <- 直接放弃作答
+case-004: json                      <- 声明过的 JSON 契约没兑现
+case-007: min_chars
+case-011: min_chars
+```
+
+---
+
+## 它怎么挑用例
+
+```mermaid
+flowchart LR
+    A[JSONL 日志] --> B[解析与归一化]
+    B --> C[计算信号得分]
+    C --> D[近重复聚类]
+    D --> E{分数高于<br/>最低分?}
+    E -- 否 --> F[丢弃]
+    E -- 是 --> G[构建用例]
+    G --> H[cases.jsonl<br/>report.md]
+    H --> I[对输出评分]
+    I --> J[metrics.json]
+    J --> K{与基线对比}
+    K -- 有回归 --> L[退出码 1]
+    K -- 干净 --> M[退出码 0]
+```
+
+每条 trace 按信号加权重打分，分数决定谁被提拔。信号和权重就在 `signals.py` 顶部：
+
+| 信号 | 权重 | 为什么它算信号 |
+| --- | --- | --- |
+| `negative_feedback` | 3.0 | 用户已经告诉你答错了。 |
+| `empty_output` | 3.0 | 什么都没返回。 |
+| `expected_shape_violated` | 3.0 | 声明过的契约（比如 JSON）没被满足。 |
+| `user_retried` | 2.5 | 他问了两遍，说明第一次没解决。 |
+| `fallback_phrase` | 2.0 | 模型放弃回答，而不是作答。 |
+| `explicit_expectation` | 2.0 | 这条 trace 自己就带了断言。 |
+| `output_much_shorter` | 1.5 | 大概率被截断，和日志中位数比。 |
+| `output_much_longer` | 1.0 | 通常是不受控的长篇大论。 |
+| `slow_response` | 1.0 | 超过整份日志的 p95 延迟。 |
+| `expensive_call` | 1.0 | 超过整份日志的 p95 成本。 |
+
+设计取舍的完整论证在 [`DECISIONS.md`](DECISIONS.md)。
+
+---
+
+## 输入格式
+
+每行一个 JSON 对象，只有 `input` 和 `output` 必填，常见别名（`prompt`/`response`、`question`/`completion` 等）也认。
+
+```json
+{
+  "id": "req-0042",
+  "input": "账单为什么变多了",
+  "output": "账单增加通常是因为套餐在到期后自动续费……",
+  "latency_ms": 830,
+  "cost_usd": 0.00027,
+  "feedback": "negative",
+  "retried": true,
+  "expect": { "json": true, "required": ["status"], "contains": ["已受理"] }
+}
+```
+
+格式坏掉的行、没有输入的行会跳过并计数。空的 `output` 不算坏行——它是日志里最有价值的行之一。
+
+---
+
+## 已知边界
+
+- **相似度是字符 n-gram**，不含相同字符的改写会被当成两个问题。可用 `--similarity` 换成你自己的实现。
+- **行为型失败抓不住。** 样本日志 16 个用例里有 3 个，失败原因只是"用户又问了一遍"，输出本身没毛病——任何对输出文本的确定性检查都抓不到。报告里已单独标出。
+- **比较成本是 O(用例数 × 每簇不同问法数)**，不是 O(日志大小)：同一问题问 5000 次只留 1 个待比对指纹。`MAX_DISTINCT_FINGERPRINTS = 512` 是拍的保护值，没压测过。
+- **用例仍是提案**，但会自检：`report.md` 里单列"需要人看一眼"的用例。
+
+---
+
+## 仓库结构
+
+```
+src/trace2eval/
+  schema.py    trace 加载、字段别名、容错解析
+  signals.py   一条 trace 为什么值得测
+  select.py    相似度、聚类、用例构建
+  checks.py    确定性输出检查
+  runner.py    评分与回归对比
+  report.py    markdown 渲染
+  matchers.py  可替换的相似度实现
+tests/         42 个测试
+examples/      一份 42 行样本日志 + 一次基线 / 一次回归运行
+evalset/       提交进仓库的构建产物
+```
+
+## 开发
+
+```bash
+pip install -e ".[dev]"
+pytest
+```
+
+## 许可证
+
+MIT

trace2eval_cli-0.2.1.dist-info/RECORD

@@ -0,0 +1,16 @@
+trace2eval/__init__.py,sha256=U4FZFVE_8PlvsvDu36hNam03SP4eshEV3V6QIHR5cL8,1833
+trace2eval/__main__.py,sha256=3-8jUqJ56SMv1AO_k9zL-n4m2thfa83lqjYyFfglvbE,198
+trace2eval/checks.py,sha256=u2ZuguIZa5pQXnmwtpxNtGIyCHyFOZZuKeDPWgsRuSc,5456
+trace2eval/cli.py,sha256=8qe378FOkzqVo6c8G48c4Il9ypY5c4Lyn5oXjZ6rrWo,10627
+trace2eval/matchers.py,sha256=mvYM7D1hSd7MjlYya97KpVX8BEF58JbmN5-gDmJ7Clg,4079
+trace2eval/report.py,sha256=sMWIZOS9Yx-4X5vF__0LSnP35JI02SYR9Jx8GkdPr8g,9207
+trace2eval/runner.py,sha256=W33eWThu_Ray0xh7vf8TX_K31CE9FRmjlkIlwWa4N74,8216
+trace2eval/schema.py,sha256=AFBDtOVjoe98Dtg8PvGAVXXJxByz54OzVhc5LpKmnYE,7028
+trace2eval/select.py,sha256=8tuMFu6kxE9VP3TCATCyEWCK0m3tK-Zyf6EqP9U-acA,27403
+trace2eval/signals.py,sha256=wUWr8JYHblx3lRz2ugfPb8Mgat3wEB8ck45XYi95B10,6857
+trace2eval_cli-0.2.1.dist-info/licenses/LICENSE,sha256=V3e-V4qN_C-iuTVsa-1pRUrxkrPLWTLpqFuhS896Z_Q,1063
+trace2eval_cli-0.2.1.dist-info/METADATA,sha256=YVbZJT1pYBQlZdYK7jzwVQ2Ntoz4YNCRNE89Tz4kndc,8652
+trace2eval_cli-0.2.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+trace2eval_cli-0.2.1.dist-info/entry_points.txt,sha256=KDIbfvCzPM3uUMSFZF1-JJhlyrOZHO3SLut02AFY9gQ,51
+trace2eval_cli-0.2.1.dist-info/top_level.txt,sha256=UC0TN9YklI4pHkU46JR2YZLrBqzkWXkx6njiHyxdSls,11
+trace2eval_cli-0.2.1.dist-info/RECORD,,

trace2eval_cli-0.2.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

trace2eval_cli-0.2.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+trace2eval = trace2eval.cli:main

trace2eval_cli-0.2.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 rfioly
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

trace2eval_cli-0.2.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+trace2eval