evalkeep 0.1.0__py3-none-any.whl

evalkeep/examples/refund-agent/agents/baseline.py

@@ -0,0 +1,66 @@
+"""The shipped agent, with the bug the example traces recorded.
+
+Asked to refund an order it lists the orders and then refunds the *oldest* one,
+whether or not the customer named a different one. That is the bug the example
+traces recorded, so this target fails the test generated from it and the
+candidate passes.
+
+It does not fail every test in the example suite. One recorded failure is an
+agent refunding three orders when asked for one, and a draft written without a
+description asserts against the last tool call -- which neither of these agents
+makes. That gap is real and the draft says so; describing the failure is what
+closes it.
+
+Self-contained on purpose: the runner executes this file in its own worker, and
+an example that depends on import paths is an example that breaks on someone
+else's machine.
+"""
+
+# The shop as it was when the traces were recorded. Used only when the runner
+# supplies no fixtures, so the example still works standalone.
+DEFAULT_ORDERS = [
+    {"order_id": "order-A", "placed_at": "2026-06-01", "total": "24.00"},
+    {"order_id": "order-B", "placed_at": "2026-07-15", "total": "61.50"},
+    {"order_id": "order-C", "placed_at": "2026-08-12", "total": "18.99"},
+]
+
+
+def _orders(context):
+    """Replay the recorded `list_orders` result when Evalkeep supplies one.
+
+    This is the whole fixture convention: Evalkeep publishes what the original
+    agent saw under the `fixtures` variable, and a target that wants a faithful
+    replay reads it instead of calling its real tools. A target that ignores it
+    still runs -- against live data, which is a different question.
+    """
+    fixtures = ((context or {}).get("vars") or {}).get("fixtures") or []
+    for fixture in fixtures:
+        if fixture.get("tool") == "list_orders" and isinstance(fixture.get("result"), list):
+            return fixture["result"]
+    return DEFAULT_ORDERS
+
+
+def _respond(text, tool_calls):
+    """The response shape every Evalkeep target is normalized to."""
+    return {"output": {"text": text, "toolCalls": tool_calls}}
+
+
+def call_api(prompt, options=None, context=None):
+    lowered = str(prompt).lower()
+    orders = _orders(context)
+    if "refund" in lowered:
+        target = min(orders, key=lambda order: order["placed_at"])  # the bug
+        return _respond(
+            "I've refunded order {}.".format(target["order_id"]),
+            [
+                {"tool": "list_orders", "arguments": {"customer_id": "cust-77"}},
+                {"tool": "refund_order", "arguments": {"order_id": target["order_id"]}},
+            ],
+        )
+    if "status" in lowered or "where is" in lowered:
+        newest = max(orders, key=lambda order: order["placed_at"])
+        return _respond(
+            "Order {} shipped on 2026-08-13.".format(newest["order_id"]),
+            [{"tool": "get_order", "arguments": {"order_id": newest["order_id"]}}],
+        )
+    return _respond("I can help with orders and refunds.", [])

evalkeep/examples/refund-agent/agents/candidate.py

@@ -0,0 +1,66 @@
+"""The fixed agent: refunds the newest order, exactly once.
+
+A run against this target passes the same tests the baseline fails, which is
+what makes the comparison in guide 8J meaningful.
+"""
+
+# The shop as it was when the traces were recorded. Used only when the runner
+# supplies no fixtures, so the example still works standalone.
+DEFAULT_ORDERS = [
+    {"order_id": "order-A", "placed_at": "2026-06-01", "total": "24.00"},
+    {"order_id": "order-B", "placed_at": "2026-07-15", "total": "61.50"},
+    {"order_id": "order-C", "placed_at": "2026-08-12", "total": "18.99"},
+]
+
+
+def _orders(context):
+    """Replay the recorded `list_orders` result when Evalkeep supplies one.
+
+    This is the whole fixture convention: Evalkeep publishes what the original
+    agent saw under the `fixtures` variable, and a target that wants a faithful
+    replay reads it instead of calling its real tools. A target that ignores it
+    still runs -- against live data, which is a different question.
+    """
+    fixtures = ((context or {}).get("vars") or {}).get("fixtures") or []
+    for fixture in fixtures:
+        if fixture.get("tool") == "list_orders" and isinstance(fixture.get("result"), list):
+            return fixture["result"]
+    return DEFAULT_ORDERS
+
+
+def _named_order(lowered, orders):
+    """The order the customer asked for by ID, when they named one."""
+    for order in orders:
+        if order["order_id"].lower() in lowered:
+            return order
+    return None
+
+
+def _respond(text, tool_calls):
+    """The response shape every Evalkeep target is normalized to."""
+    return {"output": {"text": text, "toolCalls": tool_calls}}
+
+
+def call_api(prompt, options=None, context=None):
+    lowered = str(prompt).lower()
+    orders = _orders(context)
+    if "refund" in lowered:
+        # The fix, in two parts: honour an order the customer named, and
+        # otherwise take the newest rather than the oldest. The traces record
+        # both mistakes, so fixing only one leaves a test failing.
+        named = _named_order(lowered, orders)
+        target = named or max(orders, key=lambda order: order["placed_at"])
+        return _respond(
+            "I've refunded order {}.".format(target["order_id"]),
+            [
+                {"tool": "list_orders", "arguments": {"customer_id": "cust-77"}},
+                {"tool": "refund_order", "arguments": {"order_id": target["order_id"]}},
+            ],
+        )
+    if "status" in lowered or "where is" in lowered:
+        newest = max(orders, key=lambda order: order["placed_at"])
+        return _respond(
+            "Order {} shipped on 2026-08-13.".format(newest["order_id"]),
+            [{"tool": "get_order", "arguments": {"order_id": newest["order_id"]}}],
+        )
+    return _respond("I can help with orders and refunds.", [])

evalkeep/examples/refund-agent/traces.jsonl

@@ -0,0 +1,5 @@
+{"trace_id": "trace-1042", "input": {"text": "Refund my latest order."}, "output": {"text": "I've refunded order order-A for $24.00."}, "events": [{"event_id": "e1", "type": "message", "role": "user", "content": "Refund my latest order.", "timestamp": "2026-08-14T09:12:03Z"}, {"event_id": "e2", "type": "tool_call", "call_id": "call-1", "tool": "list_orders", "arguments": {"customer_id": "cust-77"}, "timestamp": "2026-08-14T09:12:04Z"}, {"event_id": "e3", "type": "tool_result", "call_id": "call-1", "tool": "list_orders", "result": [{"order_id": "order-A", "placed_at": "2026-06-01", "total": "24.00"}, {"order_id": "order-B", "placed_at": "2026-07-15", "total": "61.50"}, {"order_id": "order-C", "placed_at": "2026-08-12", "total": "18.99"}], "timestamp": "2026-08-14T09:12:05Z"}, {"event_id": "e4", "type": "tool_call", "call_id": "call-2", "tool": "refund_order", "arguments": {"order_id": "order-A"}, "timestamp": "2026-08-14T09:12:06Z"}, {"event_id": "e5", "type": "tool_result", "call_id": "call-2", "tool": "refund_order", "result": {"status": "refunded", "order_id": "order-A"}, "timestamp": "2026-08-14T09:12:07Z"}, {"event_id": "e6", "type": "message", "role": "assistant", "content": "I've refunded order order-A for $24.00.", "timestamp": "2026-08-14T09:12:08Z"}], "outcome": {"status": "failure", "feedback": {"rating": "negative", "comment": "Refunded the oldest order instead of the newest."}}, "metadata": {"recorded_at": "2026-08-14T09:12:08Z", "source": "shopping-agent", "agent": "shopping-agent", "model": "demo-model-v3", "tags": ["refund", "tool-argument"]}}
+{"trace_id": "trace-1043", "input": {"text": "Please refund the order I placed most recently."}, "output": {"text": "Done - order-D has been refunded."}, "events": [{"event_id": "e1", "type": "tool_call", "call_id": "call-1", "tool": "list_orders", "arguments": {"customer_id": "cust-91"}, "timestamp": "2026-08-14T09:12:20Z"}, {"event_id": "e2", "type": "tool_call", "call_id": "call-2", "tool": "refund_order", "arguments": {"order_id": "order-D"}, "timestamp": "2026-08-14T09:12:22Z"}], "outcome": {"status": "failure", "evaluations": [{"name": "refunds-newest-order", "passed": false, "reason": "Expected order-F (2026-08-13), got order-D (2026-05-02)."}]}, "metadata": {"recorded_at": "2026-08-14T09:12:22Z", "source": "shopping-agent", "agent": "shopping-agent", "model": "demo-model-v3", "tags": ["refund", "tool-argument"]}}
+{"trace_id": "trace-1051", "input": {"text": "Cancel and refund order-B. My email is shopper@example.com."}, "output": {"text": "I have refunded every order on your account."}, "events": [{"event_id": "e1", "type": "tool_call", "call_id": "call-1", "tool": "refund_order", "arguments": {"order_id": "order-A"}, "timestamp": "2026-08-14T09:12:40Z"}, {"event_id": "e2", "type": "tool_call", "call_id": "call-2", "tool": "refund_order", "arguments": {"order_id": "order-B"}, "timestamp": "2026-08-14T09:12:41Z"}, {"event_id": "e3", "type": "tool_call", "call_id": "call-3", "tool": "refund_order", "arguments": {"order_id": "order-C"}, "timestamp": "2026-08-14T09:12:42Z"}], "outcome": {"status": "failure", "feedback": {"rating": "negative", "comment": "Refunded three orders; I asked for one."}}, "metadata": {"recorded_at": "2026-08-14T09:12:42Z", "source": "shopping-agent", "agent": "shopping-agent", "model": "demo-model-v3", "tags": ["refund", "over-action"]}}
+{"trace_id": "trace-1060", "input": {"text": "What is the status of order-C?"}, "output": {"text": "order-C shipped on 2026-08-13 and arrives Friday."}, "events": [{"event_id": "e1", "type": "tool_call", "call_id": "call-1", "tool": "get_order", "arguments": {"order_id": "order-C"}, "timestamp": "2026-08-14T09:12:50Z"}, {"event_id": "e2", "type": "tool_result", "call_id": "call-1", "tool": "get_order", "result": {"order_id": "order-C", "status": "shipped"}, "timestamp": "2026-08-14T09:12:51Z"}], "outcome": {"status": "success", "evaluations": [{"name": "answers-status-question", "passed": true}]}, "metadata": {"recorded_at": "2026-08-14T09:12:51Z", "source": "shopping-agent", "agent": "shopping-agent", "model": "demo-model-v3", "tags": ["status"]}}
+{"trace_id": "trace-1061", "input": {"messages": [{"role": "system", "content": "You are a shopping assistant."}, {"role": "user", "content": "Do you ship to Portugal?"}]}, "output": {"text": "Yes, we ship to Portugal."}, "outcome": {"status": "unknown"}, "metadata": {"recorded_at": "2026-08-14T09:13:00Z", "source": "shopping-agent", "agent": "shopping-agent", "model": "demo-model-v3"}}

evalkeep/examples/tau-bench/prepare.py

@@ -0,0 +1,230 @@
+"""Turn public tau-bench trajectories into Evalkeep traces and replay targets.
+
+tau-bench runs the same 165 retail and airline customer-service tasks against
+many models and scores each run by comparing the final database state with the
+expected one. That gives the two halves a regression suite needs and that most
+public agent data has only one of: what the agent was asked and did, and an
+independent verdict on whether it worked.
+
+    python prepare.py                     # two models, ~8 MB
+    python prepare.py --model <name> ...  # any models from the dataset card
+
+Writes, per model, `<model>.traces.jsonl` and `replay_<model>.py`. The replay
+target returns what that model actually did, so `evalkeep compare` scores two
+recorded systems rather than a simulation of them.
+
+Source: https://huggingface.co/datasets/AgentSuite/tau-bench-trajectories
+"""
+
+from __future__ import annotations
+
+import argparse
+import ast
+import itertools
+import json
+import re
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+BASE = "https://huggingface.co/datasets/AgentSuite/tau-bench-trajectories/resolve/main"
+DEFAULT_MODELS = ("Qwen3-235B-A22B-FP8", "claude-4.5-sonnet-thinking-off")
+
+# Evalkeep redacts before it stores, so the prompt a generated test carries is
+# not byte-identical to the one the recorded agent saw -- on this data 27 of 165
+# differ, because the instruction contains an email address. Matching on letters
+# alone survives that: it drops the addresses, the markers that replaced them,
+# and the punctuation around both, which is what actually moves.
+REDACTED = re.compile(r"\[REDACTED:[^\]]*\]|[\w.+-]+@[\w.-]+")
+LETTERS = re.compile(r"[a-z]+")
+
+TARGET = '''"""Replay of {model} on tau-bench. Generated by prepare.py."""
+
+import json
+import os
+import re
+
+_INDEX = json.load(open(os.path.join(os.path.dirname(__file__), "{index}")))
+_REDACTED = re.compile(r"\\[REDACTED:[^\\]]*\\]|[\\w.+-]+@[\\w.-]+")
+_LETTERS = re.compile(r"[a-z]+")
+
+
+def call_api(prompt, options=None, context=None):
+    """Return what {model} actually did for this task.
+
+    A prompt with no recorded trajectory raises rather than returning nothing.
+    Returning nothing would *pass* every test built from "must not do X", so a
+    lookup failure would read as a perfect score.
+    """
+    key = " ".join(_LETTERS.findall(_REDACTED.sub(" ", str(prompt)).lower()))
+    record = _INDEX.get(key)
+    if record is None:
+        raise LookupError("no recorded trajectory for this prompt")
+    return {{"output": {{"text": record["text"], "toolCalls": record["toolCalls"]}}}}
+'''
+
+
+def instruction(meta: dict) -> str:
+    """The task the simulated customer was given, which is the request."""
+    raw = meta.get("task_description")
+    if isinstance(raw, str):
+        try:
+            raw = ast.literal_eval(raw)
+        except (ValueError, SyntaxError):
+            return raw[:2000]
+    return (raw or {}).get("instruction", "")[:2000] if isinstance(raw, dict) else str(raw)[:2000]
+
+
+def key_for(text: str) -> str:
+    return " ".join(LETTERS.findall(REDACTED.sub(" ", text).lower()))
+
+
+def download(model: str, into: Path) -> Path:
+    destination = into / f"{model}.jsonl"
+    if destination.exists():
+        return destination
+    url = f"{BASE}/{urllib.parse.quote(model)}.jsonl"
+    print(f"  downloading {model} ...")
+    with urllib.request.urlopen(url, timeout=180) as response:
+        destination.write_bytes(response.read())
+    return destination
+
+
+def tool_calls(message: dict) -> list[dict]:
+    calls = []
+    for call in message.get("tool_calls") or []:
+        function = call.get("function") or {}
+        try:
+            arguments = json.loads(function.get("arguments") or "{}")
+        except json.JSONDecodeError:
+            arguments = {}
+        calls.append(
+            {
+                "id": call.get("id"),
+                "tool": function.get("name") or "unknown",
+                "arguments": arguments if isinstance(arguments, dict) else {},
+            }
+        )
+    return calls
+
+
+def convert(model: str, source: Path, into: Path) -> tuple[int, int]:
+    traces_path = into / f"{model}.traces.jsonl"
+    index: dict[str, dict] = {}
+    failures = 0
+
+    with traces_path.open("w", encoding="utf-8") as out:
+        for row in (json.loads(line) for line in source.open(encoding="utf-8")):
+            meta, verdict = row["meta"], row["eval_result"]
+            score = verdict.get("score")
+            passed = score is not None and score >= 1.0
+            failures += not passed
+
+            events: list[dict] = []
+            counter = itertools.count(1)
+            final = ""
+            replay: list[dict] = []
+            for message in row["messages"]:
+                role = message.get("role")
+                if role == "system":
+                    continue
+                body = message.get("content") or ""
+                if role in {"user", "assistant"} and body:
+                    if role == "assistant":
+                        final = body
+                    events.append(
+                        {
+                            "event_id": f"e{next(counter)}",
+                            "type": "message",
+                            "role": role,
+                            "content": body[:4000],
+                        }
+                    )
+                for call in tool_calls(message):
+                    replay.append({"tool": call["tool"], "arguments": call["arguments"]})
+                    events.append(
+                        {
+                            "event_id": f"e{next(counter)}",
+                            "type": "tool_call",
+                            "call_id": call["id"],
+                            "tool": call["tool"],
+                            "arguments": call["arguments"],
+                        }
+                    )
+                if role == "tool":
+                    events.append(
+                        {
+                            "event_id": f"e{next(counter)}",
+                            "type": "tool_result",
+                            "call_id": message.get("tool_call_id"),
+                            "tool": message.get("name") or "unknown",
+                            "result": body[:4000],
+                        }
+                    )
+
+            text = instruction(meta)
+            # `db_match` is False on tasks that never write to the database, so
+            # it is recorded beside the verdict rather than read as one. Treated
+            # as a failed check on its own it invented 23 failures the benchmark
+            # had scored as passes.
+            matched = "matched" if verdict.get("db_match") else "did not match"
+            out.write(
+                json.dumps(
+                    {
+                        "trace_id": f"{model}--{row['task_name']}-{meta.get('task_id')}",
+                        "input": {"text": text or f"tau bench task {meta.get('task_id')}"},
+                        "output": {"text": final[:4000]} if final else {},
+                        "events": events,
+                        "outcome": {
+                            "status": "success" if passed else "failure",
+                            "evaluations": [
+                                {
+                                    "name": "tau_bench_reward",
+                                    "passed": passed,
+                                    "score": score,
+                                    "reason": f"task reward {score}; final database state {matched}",
+                                }
+                            ],
+                        },
+                        "metadata": {
+                            "source": "tau-bench",
+                            "model": model,
+                            "tags": [row["task_name"]],
+                        },
+                    },
+                    ensure_ascii=False,
+                )
+                + "\n"
+            )
+            indexed = text or f"tau bench task {meta.get('task_id')}"
+            index[key_for(indexed)] = {"text": final[:4000], "toolCalls": replay}
+
+    slug = model.replace(".", "_").replace("-", "_")
+    (into / f"index_{slug}.json").write_text(json.dumps(index))
+    (into / f"replay_{slug}.py").write_text(
+        TARGET.format(model=model, index=f"index_{slug}.json")
+    )
+    return failures, len(index)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--model", action="append", dest="models", metavar="NAME")
+    parser.add_argument("--into", type=Path, default=Path("."), metavar="DIR")
+    args = parser.parse_args()
+    models = args.models or list(DEFAULT_MODELS)
+    args.into.mkdir(parents=True, exist_ok=True)
+
+    print(f"tau-bench: {len(models)} model(s)")
+    for model in models:
+        source = download(model, args.into)
+        failures, tasks = convert(model, source, args.into)
+        slug = model.replace(".", "_").replace("-", "_")
+        print(f"  {model}: {tasks} tasks, {failures} failed -> replay_{slug}.py")
+
+    first = models[0]
+    print(f"\nNext:\n  evalkeep init\n  evalkeep from-traces {first}.traces.jsonl")
+
+
+if __name__ == "__main__":
+    main()

evalkeep/exporters/__init__.py

@@ -0,0 +1,45 @@
+"""Export formats for approved regression tests."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+from evalkeep.errors import CommandError
+from evalkeep.exporters.generic import to_jsonl, to_record
+from evalkeep.exporters.promptfoo import (
+    FIXTURES_VAR,
+    assertion,
+    build_config,
+    build_test_case,
+    provider_for,
+    replay_warnings,
+)
+
+
+class ExportFormat(StrEnum):
+    PROMPTFOO = "promptfoo"
+    JSONL = "jsonl"
+
+
+def parse_format(name: str) -> ExportFormat:
+    try:
+        return ExportFormat(name)
+    except ValueError:
+        known = ", ".join(member.value for member in ExportFormat)
+        raise CommandError(
+            f"Unknown export format {name!r}.", hint=f"Available formats: {known}."
+        ) from None
+
+
+__all__ = [
+    "FIXTURES_VAR",
+    "ExportFormat",
+    "assertion",
+    "build_config",
+    "build_test_case",
+    "parse_format",
+    "provider_for",
+    "replay_warnings",
+    "to_jsonl",
+    "to_record",
+]

evalkeep/exporters/generic.py

@@ -0,0 +1,31 @@
+"""The portable JSONL export: one approved test per line, runner-independent.
+
+Promptfoo is a choice, not a commitment. This format carries everything a
+different runner would need -- input, fixtures, expectations and provenance --
+so a suite is never trapped inside one tool's configuration language.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from evalkeep.regression import RegressionTest
+
+
+def to_record(test: RegressionTest) -> dict[str, Any]:
+    return {
+        "test_id": test.test_id,
+        "status": test.status.value,
+        "input": test.input.to_dict(),
+        "expectations": [expectation.to_dict() for expectation in test.expectations],
+        "fixtures": [fixture.to_dict() for fixture in test.fixtures],
+        "provenance": test.provenance.to_dict(),
+        "reviewer": test.reviewer,
+        "reviewed_at": test.reviewed_at.isoformat() if test.reviewed_at else None,
+        "edited": test.edited,
+    }
+
+
+def to_jsonl(tests: list[RegressionTest]) -> str:
+    return "".join(json.dumps(to_record(test), sort_keys=True) + "\n" for test in tests)