evalkeep 0.1.0__py3-none-any.whl

evalkeep/targets.py

@@ -0,0 +1,205 @@
+"""Agent targets: how to reach the thing under test, without its credentials.
+
+A target says where an agent lives and how to read its answer. It is committed
+to Git, so it must never contain a secret -- and that is enforced, not merely
+requested: saving a target whose configuration contains something that looks
+like a credential is refused, and the same detectors that redact traces do the
+looking. Secrets are referenced as ``${ENV_VAR}`` and resolved by the runner
+from the environment at run time.
+
+Every provider kind is normalized to one response shape::
+
+    {"text": "...", "toolCalls": [{"tool": "...", "arguments": {...}}, ...]}
+
+so an expectation means the same thing whether it runs against an HTTP endpoint,
+a local script or a model provider.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from enum import StrEnum
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+
+from evalkeep.errors import CommandError
+from evalkeep.redaction import TOKEN_PATTERNS, is_secret_field
+
+TARGETS_FILENAME = "targets.yaml"
+TARGETS_VERSION = 1
+
+BASELINE = "baseline"
+CANDIDATE = "candidate"
+
+#: The only way a secret may appear in a committed target.
+ENV_REFERENCE = re.compile(r"^\$\{([A-Z][A-Z0-9_]*)\}$")
+
+
+class TargetKind(StrEnum):
+    HTTP = "http"
+    PYTHON = "python"
+    JAVASCRIPT = "javascript"
+    #: A model provider called directly, for testing tool selection rather than
+    #: a whole application.
+    MODEL = "model"
+
+
+class Extraction(BaseModel):
+    """Where the answer lives in the target's response.
+
+    Expressions are evaluated over the parsed response body, so ``json.reply``
+    reads ``{"reply": ...}``. Only HTTP targets need these; a script or model
+    provider is adapted by the generated configuration.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    output: str = "json.output"
+    tool_calls: str | None = "json.toolCalls"
+
+
+class Target(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    target_id: str
+    kind: TargetKind
+    description: str | None = None
+
+    # HTTP
+    url: str | None = None
+    method: str = "POST"
+    headers: dict[str, str] = Field(default_factory=dict)
+    body: dict[str, Any] = Field(default_factory=dict)
+    extract: Extraction = Field(default_factory=Extraction)
+
+    # Script targets
+    path: str | None = None
+    function: str | None = None
+
+    # Direct model provider
+    provider: str | None = None
+
+    def validate_shape(self) -> None:
+        """Check the fields this kind of target actually needs."""
+        match self.kind:
+            case TargetKind.HTTP:
+                if not self.url:
+                    raise CommandError("An http target needs a url.")
+                if not self.body:
+                    raise CommandError(
+                        "An http target needs a body.",
+                        hint="Use {{input}} where the test input should go.",
+                    )
+            case TargetKind.PYTHON | TargetKind.JAVASCRIPT:
+                if not self.path:
+                    raise CommandError(f"A {self.kind.value} target needs a path.")
+            case TargetKind.MODEL:
+                if not self.provider:
+                    raise CommandError(
+                        "A model target needs a provider.",
+                        hint="For example: anthropic:messages:claude-opus-5",
+                    )
+
+
+class TargetFile(BaseModel):
+    """The contents of ``targets.yaml``."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    version: int = TARGETS_VERSION
+    targets: dict[str, Target] = Field(default_factory=dict)
+
+    def to_yaml(self) -> str:
+        payload = self.model_dump(mode="json", exclude_none=True)
+        for target in payload["targets"].values():
+            target.pop("target_id", None)
+        return yaml.safe_dump(payload, sort_keys=False, default_flow_style=False)
+
+
+def find_secrets(target: Target) -> list[str]:
+    """Every place this target holds a literal credential.
+
+    Two rules, both borrowed from redaction so the definitions cannot drift: a
+    value that *looks* like a known token, and any value under a
+    credential-shaped key that is not an ``${ENV_VAR}`` reference.
+    """
+    problems: list[str] = []
+    for path, value in _walk(target.model_dump(mode="json")):
+        if not isinstance(value, str) or ENV_REFERENCE.match(value.strip()):
+            continue
+        key = path.rsplit(".", 1)[-1]
+        if any(pattern.search(value) for pattern in TOKEN_PATTERNS):
+            problems.append(f"{path} looks like a credential")
+        elif is_secret_field(key) and value.strip():
+            problems.append(
+                f"{path} is a credential field with a literal value; use ${{ENV_VAR}} instead"
+            )
+    return problems
+
+
+def referenced_environment(target: Target) -> dict[str, bool]:
+    """Which environment variables this target needs, and whether each is set."""
+    referenced: dict[str, bool] = {}
+    for _, value in _walk(target.model_dump(mode="json")):
+        if isinstance(value, str) and (match := ENV_REFERENCE.match(value.strip())):
+            name = match.group(1)
+            referenced[name] = bool(os.environ.get(name))
+    return referenced
+
+
+def load_targets(root: Path) -> TargetFile:
+    path = root / TARGETS_FILENAME
+    if not path.is_file():
+        return TargetFile()
+    try:
+        raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    except yaml.YAMLError as exc:
+        raise CommandError(f"Could not parse {path}: {exc}") from exc
+    if not isinstance(raw, dict):
+        raise CommandError(f"Expected a mapping in {path}.")
+
+    for target_id, entry in (raw.get("targets") or {}).items():
+        if isinstance(entry, dict):
+            entry.setdefault("target_id", target_id)
+    try:
+        return TargetFile.model_validate(raw)
+    except ValidationError as exc:
+        raise CommandError(f"Invalid target configuration in {path}:\n{exc}") from exc
+
+
+def save_targets(root: Path, targets: TargetFile) -> Path:
+    path = root / TARGETS_FILENAME
+    try:
+        path.write_text(targets.to_yaml(), encoding="utf-8")
+    except OSError as exc:
+        raise CommandError(f"Could not write {path}: {exc}") from exc
+    return path
+
+
+def get_target(root: Path, target_id: str) -> Target:
+    targets = load_targets(root)
+    target = targets.targets.get(target_id.strip())
+    if target is None:
+        known = ", ".join(sorted(targets.targets)) or "none configured"
+        raise CommandError(
+            f"No target named {target_id.strip()!r}.",
+            hint=f"Known targets: {known}. Add one with 'evalkeep targets add'.",
+        )
+    return target
+
+
+def _walk(value: Any, prefix: str = "") -> list[tuple[str, Any]]:
+    found: list[tuple[str, Any]] = []
+    if isinstance(value, dict):
+        for key, child in value.items():
+            found.extend(_walk(child, f"{prefix}{key}."))
+    elif isinstance(value, list):
+        for index, child in enumerate(value):
+            found.extend(_walk(child, f"{prefix}{index}."))
+    else:
+        found.append((prefix.rstrip("."), value))
+    return found

evalkeep/trace.py

@@ -0,0 +1,238 @@
+"""The normalized trace schema.
+
+Every adapter converts a provider's format into these models, so the rest of
+Evalkeep -- redaction, failure detection, clustering, test generation -- only
+ever sees one shape.
+
+Two deliberate strictnesses:
+
+* Structural models reject unknown fields. A typo like ``trace-id`` is a defect
+  worth reporting, not data worth keeping, and every field that survives here
+  must be one the redactor knows how to scrub. Arbitrary provider data belongs
+  under ``metadata.extra``, which redaction walks recursively.
+* Identifiers are stripped and must be non-empty, so a whitespace-only
+  ``trace_id`` fails validation rather than becoming a silent duplicate.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from enum import StrEnum
+from typing import Annotated, Any, Literal
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    StringConstraints,
+    field_validator,
+    model_validator,
+)
+
+#: Bumped when the trace schema changes in a way stored traces must migrate for.
+SCHEMA_VERSION = 1
+
+MAX_IDENTIFIER_LENGTH = 256
+MAX_TOOL_NAME_LENGTH = 128
+
+#: Identifiers are trimmed and must survive trimming.
+Identifier = Annotated[
+    str,
+    StringConstraints(strip_whitespace=True, min_length=1, max_length=MAX_IDENTIFIER_LENGTH),
+]
+
+#: Tool names must look like callable identifiers: no spaces, no path separators,
+#: nothing that could be mistaken for a shell fragment when exported to a runner.
+ToolName = Annotated[
+    str,
+    StringConstraints(
+        strip_whitespace=True,
+        min_length=1,
+        max_length=MAX_TOOL_NAME_LENGTH,
+        pattern=r"^[A-Za-z_][A-Za-z0-9_.-]*$",
+    ),
+]
+
+NonEmptyStr = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)]
+
+
+class Role(StrEnum):
+    USER = "user"
+    ASSISTANT = "assistant"
+    SYSTEM = "system"
+    TOOL = "tool"
+
+
+class OutcomeStatus(StrEnum):
+    SUCCESS = "success"
+    FAILURE = "failure"
+    ERROR = "error"
+    UNKNOWN = "unknown"
+
+
+class _Strict(BaseModel):
+    """Base for every trace model: unknown fields are a validation error."""
+
+    model_config = ConfigDict(extra="forbid")
+
+
+def _as_utc(value: datetime | None) -> datetime | None:
+    """Treat a naive timestamp as UTC so orderings stay comparable."""
+    if value is not None and value.tzinfo is None:
+        return value.replace(tzinfo=UTC)
+    return value
+
+
+class Message(_Strict):
+    role: Role
+    content: str
+
+
+class TraceInput(_Strict):
+    """What the agent was asked to do."""
+
+    text: str | None = None
+    messages: list[Message] = Field(default_factory=list)
+
+    @model_validator(mode="after")
+    def _require_content(self) -> TraceInput:
+        if not (self.text or "").strip() and not self.messages:
+            raise ValueError("input must provide non-empty 'text' or at least one message")
+        return self
+
+
+class TraceOutput(_Strict):
+    """What the agent produced. Absent for traces captured before a response."""
+
+    text: str | None = None
+    messages: list[Message] = Field(default_factory=list)
+
+
+class _BaseEvent(_Strict):
+    event_id: Identifier
+    timestamp: datetime | None = None
+
+    _normalize_timestamp = field_validator("timestamp")(_as_utc)
+
+
+class MessageEvent(_BaseEvent):
+    type: Literal["message"] = "message"
+    role: Role
+    content: str
+
+
+class ToolCallEvent(_BaseEvent):
+    type: Literal["tool_call"] = "tool_call"
+    tool: ToolName
+    arguments: dict[str, Any] = Field(default_factory=dict)
+    call_id: Identifier | None = None
+
+
+class ToolResultEvent(_BaseEvent):
+    type: Literal["tool_result"] = "tool_result"
+    tool: ToolName
+    call_id: Identifier | None = None
+    result: Any = None
+    error: str | None = None
+
+
+class EvaluationEvent(_BaseEvent):
+    """An evaluator's verdict recorded alongside the interaction."""
+
+    type: Literal["evaluation"] = "evaluation"
+    name: NonEmptyStr
+    passed: bool | None = None
+    score: float | None = None
+    reason: str | None = None
+
+
+Event = Annotated[
+    MessageEvent | ToolCallEvent | ToolResultEvent | EvaluationEvent,
+    Field(discriminator="type"),
+]
+
+
+class Feedback(_Strict):
+    """Human or automated feedback attached to the interaction."""
+
+    rating: Literal["positive", "negative"] | None = None
+    score: float | None = None
+    comment: str | None = None
+
+
+class Evaluation(_Strict):
+    name: NonEmptyStr
+    passed: bool | None = None
+    score: float | None = None
+    reason: str | None = None
+
+
+class Outcome(_Strict):
+    """The three evidence sources the failure detectors read in guide 8D."""
+
+    status: OutcomeStatus = OutcomeStatus.UNKNOWN
+    feedback: Feedback | None = None
+    evaluations: list[Evaluation] = Field(default_factory=list)
+
+
+class TraceMetadata(_Strict):
+    recorded_at: datetime | None = None
+    source: str | None = None
+    agent: str | None = None
+    model: str | None = None
+    tags: list[str] = Field(default_factory=list)
+    #: The one open field. Provider-specific data goes here and is redacted
+    #: recursively; nothing else in the schema accepts unknown keys.
+    extra: dict[str, Any] = Field(default_factory=dict)
+
+    _normalize_recorded_at = field_validator("recorded_at")(_as_utc)
+
+
+class NormalizedTrace(_Strict):
+    """One recorded interaction, in the only shape Evalkeep stores."""
+
+    schema_version: int = SCHEMA_VERSION
+    trace_id: Identifier
+    input: TraceInput
+    output: TraceOutput | None = None
+    events: list[Event] = Field(default_factory=list)
+    outcome: Outcome = Field(default_factory=Outcome)
+    metadata: TraceMetadata = Field(default_factory=TraceMetadata)
+
+    @model_validator(mode="after")
+    def _validate_event_sequence(self) -> NormalizedTrace:
+        """Events must be uniquely identified, ordered, and internally consistent."""
+        seen_event_ids: set[str] = set()
+        open_calls: set[str] = set()
+        previous: datetime | None = None
+
+        for index, event in enumerate(self.events):
+            where = f"events[{index}]"
+            if event.event_id in seen_event_ids:
+                raise ValueError(f"{where}: duplicate event_id {event.event_id!r}")
+            seen_event_ids.add(event.event_id)
+
+            if event.timestamp is not None:
+                if previous is not None and event.timestamp < previous:
+                    raise ValueError(
+                        f"{where}: timestamp {event.timestamp.isoformat()} is earlier than "
+                        f"the preceding event's {previous.isoformat()}; events must be ordered"
+                    )
+                previous = event.timestamp
+
+            if isinstance(event, ToolCallEvent) and event.call_id is not None:
+                if event.call_id in open_calls:
+                    raise ValueError(f"{where}: duplicate call_id {event.call_id!r}")
+                open_calls.add(event.call_id)
+            elif isinstance(event, ToolResultEvent) and event.call_id is not None:
+                if event.call_id not in open_calls:
+                    raise ValueError(
+                        f"{where}: call_id {event.call_id!r} does not match an earlier tool_call"
+                    )
+
+        return self
+
+    @property
+    def tool_calls(self) -> list[ToolCallEvent]:
+        """Observed tool calls, in order. Used by detection and test generation."""
+        return [event for event in self.events if isinstance(event, ToolCallEvent)]

evalkeep-0.1.0.dist-info/METADATA

@@ -0,0 +1,221 @@
+Metadata-Version: 2.4
+Name: evalkeep
+Version: 0.1.0
+Summary: Turn production agent failures into a small, reviewed regression suite.
+Keywords: llm,evals,evaluation,regression-testing,ai-agents,testing,promptfoo
+Author: Rakshita Devurkar
+Author-email: Rakshita Devurkar <13130544+rakshita-devurkar@users.noreply.github.com>
+License-Expression: Apache-2.0
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Requires-Dist: click>=8.5.0
+Requires-Dist: numpy>=2.4.6
+Requires-Dist: pydantic>=2.13.5
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: rich>=15.0.0
+Requires-Dist: typer>=0.27.2
+Requires-Dist: anthropic>=1.0.0 ; extra == 'anthropic'
+Requires-Python: >=3.11
+Project-URL: Homepage, https://github.com/rakshita-devurkar/evalkeep
+Project-URL: Repository, https://github.com/rakshita-devurkar/evalkeep
+Project-URL: Issues, https://github.com/rakshita-devurkar/evalkeep/issues
+Project-URL: Changelog, https://github.com/rakshita-devurkar/evalkeep/blob/main/CHANGELOG.md
+Project-URL: Documentation, https://github.com/rakshita-devurkar/evalkeep/blob/main/docs/pipeline.md
+Provides-Extra: anthropic
+Description-Content-Type: text/markdown
+
+# Evalkeep
+
+**Stop fixing the same agent bug twice.** Evalkeep turns production failures
+into a small, reviewed regression suite, and tells you whether a fix held — or
+that the evidence is too thin to say.
+
+Existing eval tools *execute* tests. The hard part is deciding which of
+thousands of production traces deserve permanent coverage. Evalkeep owns that
+decision:
+
+```
+trace → failure evidence → failure family → representative case
+      → reviewed regression test → runner execution → trustworthy comparison
+```
+
+**Evalkeep does not run your agent and is not an eval framework.** It generates
+tests, delegates execution to [Promptfoo](https://promptfoo.dev), and compares
+baseline against candidate. It sits upstream of your eval runner, not next to it.
+
+## Install
+
+```bash
+git clone https://github.com/rakshita-devurkar/evalkeep && cd evalkeep
+uv sync
+```
+
+Node.js is needed only for `evalkeep run`, which shells out to Promptfoo.
+
+## Quick start
+
+One command takes a trace file to a review queue. Offline, no API key:
+
+```bash
+uv run evalkeep demo .
+uv run evalkeep init
+uv run evalkeep from-traces refund-agent/traces.jsonl
+```
+
+```
+5 trace(s) ingested
+3 failure(s) found  explicit_status x3, failed_evaluator x1, negative_feedback x2
+2 failure famil(ies)
+3 with enough evidence for a regression test
+3 of them only forbid the mistake that was observed; say what should have happened at review.
+
+Review them: evalkeep review  (3 pending)
+```
+
+That second qualifier is the honest state of a first run: nothing in a trace
+says what the agent *should* have done. Describing the failures closes it, by
+hand at review or with an analyzer configured. It stops at review on purpose —
+approving a test is a judgement.
+
+## On real agent data
+
+The example above is five traces. Here is the same pipeline on
+[tau-bench trajectories](https://huggingface.co/datasets/AgentSuite/tau-bench-trajectories):
+165 retail and airline customer-service tasks per model, each scored by
+comparing the final database state against the expected one. Real tool calls,
+and an independent verdict — the two halves a regression suite needs.
+
+```bash
+python tau-bench/prepare.py                      # ~8 MB, two models
+uv run evalkeep from-traces tau-bench/Qwen3-235B-A22B-FP8.traces.jsonl
+```
+
+```
+165 trace(s) ingested
+90 failure(s) found  explicit_status x90, failed_evaluator x90
+4 failure famil(ies)
+```
+
+Four families, from 90 failures: retail exchange and refund flows, airline
+reservation changes, and two shapes of giving up and escalating to a human.
+Build a test per failure, approve them, and run two recorded models against the
+suite one of them produced:
+
+```bash
+uv run evalkeep dataset build --all
+uv run evalkeep review
+uv run evalkeep targets add baseline  --type python --function call_api \
+  --path tau-bench/replay_Qwen3_235B_A22B_FP8.py
+uv run evalkeep targets add candidate --type python --function call_api \
+  --path tau-bench/replay_claude_4_5_sonnet_thinking_off.py
+uv run evalkeep run --target baseline && uv run evalkeep run --target candidate
+uv run evalkeep compare
+```
+
+```
+compared                           89
+baseline pass rate               4.5%
+candidate pass rate             42.7%
+difference                     +38.2%
+p-value                        0.0000
+95% interval         +27.2% to +49.2%
+McNemar's exact test: the change is unlikely to be chance.
+
+1 test(s) excluded and not counted in any rate above.
+```
+
+Baseline scoring 4.5% is the control: the tests came from its own failures, so
+it should fail nearly all of them. The four it passes are the documented
+weakness of deriving a test with nothing describing the failure — assertions
+target the last tool call, which is sometimes a harmless lookup. The excluded
+test is one whose target raised rather than answered, and it is kept out of
+every rate rather than counted as a failure.
+
+## Doing it stage by stage
+
+`from-traces` runs five commands in order. Each is a real decision with its own
+evidence, and you will want them separately once you are tuning a suite:
+
+```bash
+uv run evalkeep ingest traces.jsonl   # validate, redact, store
+uv run evalkeep detect                # evidence-backed failures
+uv run evalkeep analyze               # describe them (or: failures label)
+uv run evalkeep discover              # embed, cluster, pick representatives
+uv run evalkeep dataset build         # draft a test per representative
+```
+
+Re-running `from-traces` is safe: it skips traces it already has and rebuilds
+drafts, but never touches a test you have reviewed.
+
+## Bring your own traces
+
+```bash
+uv run evalkeep ingest spans.json --format otlp        # OpenTelemetry / OpenInference
+uv run evalkeep ingest runs.jsonl --format langsmith   # LangSmith
+```
+
+Adapters read files, never APIs — no credentials, any vendor tier. OpenTelemetry
+covers the most ground, since Langfuse, Braintrust and Phoenix all ingest OTLP.
+`evalkeep demo` writes an example export in each format.
+
+## Commands
+
+| Stage | Commands |
+| --- | --- |
+| Set up | `init`, `targets add/list/show/remove` |
+| Ingest | `ingest`, `trace list/show` |
+| Detect | `detect`, `failures list/show/confirm/dismiss/add` |
+| Analyze | `analyze`, `failures label` |
+| Group | `discover`, `clusters list/show/rename/merge/split/dismiss/restore` |
+| Build | `dataset build/list/show` |
+| Review | `review`, `dataset approve/reject/edit` |
+| Run | `export`, `run --target ...`, `runs list/show` |
+| Compare | `compare`, `baseline promote/show` |
+
+## What it guarantees
+
+- **Values are redacted before storage** — in memory, with no path around it.
+  Identifiers can be [pseudonymized](docs/security.md#identifiers) too.
+- **Automation never overwrites human judgement.** Re-running any stage
+  refreshes derived data and leaves your reviews, labels and edits alone.
+- **Nothing is exported without approval.** Generated tests are drafts.
+- **A test that never ran is not a test that failed.** Timeouts and crashed
+  providers are excluded from every rate, so an outage cannot read as a regression.
+- **One lucky pass is not a fix.** `run --repetitions N` reports a per-case
+  verdict; a case that only sometimes passes is flaky, never passing.
+- **Score changes are not overclaimed.** McNemar's exact test, and no confidence
+  interval when the sample cannot support one.
+
+## More
+
+[How it works](docs/pipeline.md) · [Privacy and security](docs/security.md) ·
+[Roadmap](docs/roadmap.md) · [Contributing](CONTRIBUTING.md) ·
+[Changelog](CHANGELOG.md)
+
+```bash
+uv sync && uv run pytest              # 985 tests
+uv run ruff check . && uv run mypy    # lint and strict types
+```
+
+`EVALKEEP_E2E=1 uv run pytest` also runs the suite against real Promptfoo.
+Exit codes: `0` success, `1` ran but some records were rejected, `2` could not
+run.
+
+0.1 is feature-complete and not yet released to PyPI. Multi-turn replay and
+longitudinal failure history are still open; see the
+[roadmap](docs/roadmap.md).
+
+## License
+
+Apache-2.0. See [LICENSE](LICENSE).