evalkeep 0.1.0__py3-none-any.whl

evalkeep/review.py

@@ -0,0 +1,309 @@
+"""Human review: the gate between a generated draft and a committed test.
+
+Everything here is a pure function over a test. The interactive terminal loop is
+a thin shell in the CLI that calls these, which is what keeps the guide's
+"non-interactive CI review format" possible later: a different front end -- a
+web form, a PR check, a batch file of decisions -- needs no new logic, only a
+different way of collecting the same decisions.
+
+Two rules the review gate enforces rather than suggests:
+
+* **A contradictory test cannot be approved.** It would fail on a correct agent
+  too, reporting a regression that is really a bug in the suite.
+* **Editing cannot change what a test *is*.** A reviewer edits the input and the
+  expectations -- the parts that encode intent. The test ID, provenance and
+  fixtures are facts about where the test came from, and letting a review rewrite
+  them would make the audit trail describe something that never happened.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from enum import StrEnum
+from typing import Any
+
+import yaml
+
+from evalkeep.generation import NO_POSITIVE_EXPECTATION
+from evalkeep.regression import (
+    CaseInput,
+    Expectation,
+    ExpectationType,
+    RegressionTest,
+    ReviewStatus,
+    find_contradictions,
+    validate_expectation,
+)
+
+#: The only fields a reviewer may change.
+EDITABLE_FIELDS = ("input", "expectations")
+
+EDIT_HELP = """\
+# Editing {test_id}
+#
+# Change the input and the expectations. Everything else -- the test ID, the
+# provenance, the recorded fixtures -- describes where this test came from and
+# is not editable.
+#
+# Expectation types:
+#   output_contains        value: text the answer must contain
+#   output_not_contains    value: text the answer must not contain
+#   output_matches         value: a regular expression the answer must match
+#   tool_called            tool: a tool that must be called
+#   tool_not_called        tool: a tool that must never be called
+#   tool_argument_equals   tool, path, value
+#   tool_argument_not_equals   tool, path, value
+#   max_tool_calls         value: a whole number; tool: optional
+#   human_rubric           value: what should happen, judged by a model at run time
+#
+# Delete every expectation to abandon the edit.
+"""
+
+
+class ReviewDecision(StrEnum):
+    APPROVE = "approve"
+    EDIT = "edit"
+    REJECT = "reject"
+    SKIP = "skip"
+
+
+@dataclass
+class ReviewOutcome:
+    """What one review session did."""
+
+    reviewed: int = 0
+    approved: int = 0
+    rejected: int = 0
+    edited: int = 0
+    skipped: int = 0
+    remaining: int = 0
+
+    @property
+    def changed(self) -> bool:
+        return bool(self.approved or self.rejected or self.edited)
+
+
+@dataclass
+class EditResult:
+    """A parsed edit, or the reasons it could not be applied."""
+
+    test: RegressionTest | None = None
+    errors: list[str] = field(default_factory=list)
+
+    @property
+    def ok(self) -> bool:
+        return self.test is not None and not self.errors
+
+
+class ReviewError(Exception):
+    """A decision that cannot be recorded as asked."""
+
+
+def recompute_warnings(test: RegressionTest) -> list[str]:
+    """What still needs a reviewer's attention, for the test as it stands now.
+
+    Recomputed rather than carried forward: warnings describe current content,
+    and a note about how the draft was generated stops being true the moment a
+    person edits it.
+    """
+    warnings: list[str] = []
+    for expectation in test.expectations:
+        problem = validate_expectation(expectation)
+        if problem is not None:
+            warnings.append(f"Invalid expectation ({expectation.describe()}): {problem}")
+    warnings.extend(
+        f"Contradictory expectations: {contradiction.describe()}"
+        for contradiction in test.contradictions
+    )
+    if not test.has_positive_expectation:
+        warnings.append(NO_POSITIVE_EXPECTATION)
+    return warnings
+
+
+def blocking_problems(test: RegressionTest) -> list[str]:
+    """Reasons a test must not be approved as it stands."""
+    problems = [
+        f"Invalid expectation ({expectation.describe()}): {problem}"
+        for expectation in test.expectations
+        if (problem := validate_expectation(expectation)) is not None
+    ]
+    problems.extend(
+        f"Contradictory expectations: {contradiction.describe()}"
+        for contradiction in test.contradictions
+    )
+    if not test.expectations:
+        problems.append("A test with no expectations checks nothing.")
+    return problems
+
+
+def approve(test: RegressionTest, *, reviewer: str, reason: str | None = None) -> RegressionTest:
+    """Approve a test, refusing if it could never pass on a correct agent."""
+    problems = blocking_problems(test)
+    if problems:
+        raise ReviewError(
+            "This test cannot be approved as it stands:\n  - " + "\n  - ".join(problems)
+        )
+    return _record(test, ReviewStatus.APPROVED, reviewer=reviewer, reason=reason)
+
+
+def reject(test: RegressionTest, *, reviewer: str, reason: str | None = None) -> RegressionTest:
+    """Reject a test. The record is kept, not deleted: a rejection is evidence."""
+    return _record(test, ReviewStatus.REJECTED, reviewer=reviewer, reason=reason)
+
+
+def _record(
+    test: RegressionTest, status: ReviewStatus, *, reviewer: str, reason: str | None
+) -> RegressionTest:
+    test.status = status
+    test.reviewer = reviewer
+    test.review_reason = reason
+    test.reviewed_at = datetime.now(UTC)
+    test.updated_at = test.reviewed_at
+    test.warnings = recompute_warnings(test)
+    return test
+
+
+def render_editable(test: RegressionTest) -> str:
+    """The YAML a reviewer edits, with the guidance they need above it."""
+    body = {
+        "input": {
+            key: value for key, value in test.input.to_dict().items() if value not in (None, [])
+        },
+        "expectations": [expectation.to_dict() for expectation in test.expectations],
+    }
+    header = EDIT_HELP.format(test_id=test.test_id)
+    return header + yaml.safe_dump(body, sort_keys=False, default_flow_style=False)
+
+
+def apply_edits(test: RegressionTest, text: str, *, editor: str) -> EditResult:
+    """Parse an edited document and return an updated copy, or the errors.
+
+    Nothing is written until the result is valid: an unparseable or
+    self-contradictory edit leaves the stored draft exactly as it was.
+    """
+    try:
+        # safe_load, never load: this document is arbitrary text from an editor,
+        # and full YAML can construct objects.
+        raw: Any = yaml.safe_load(text)
+    except yaml.YAMLError as exc:
+        return EditResult(errors=[f"Could not parse YAML: {exc}"])
+
+    if raw is None:
+        return EditResult(errors=["The document is empty."])
+    if not isinstance(raw, dict):
+        return EditResult(errors=[f"Expected a mapping, got {type(raw).__name__}."])
+
+    unknown = set(raw) - set(EDITABLE_FIELDS)
+    if unknown:
+        return EditResult(
+            errors=[
+                f"Only {' and '.join(EDITABLE_FIELDS)} can be edited; "
+                f"remove: {', '.join(sorted(unknown))}."
+            ]
+        )
+
+    errors: list[str] = []
+    case_input, input_errors = _parse_input(raw.get("input"))
+    errors.extend(input_errors)
+    expectations, expectation_errors = _parse_expectations(raw.get("expectations"))
+    errors.extend(expectation_errors)
+
+    if errors:
+        return EditResult(errors=errors)
+
+    updated = _copy_with_edits(test, case_input, expectations, editor=editor)
+    contradictions = [
+        f"Contradictory expectations: {contradiction.describe()}"
+        for contradiction in find_contradictions(expectations)
+    ]
+    if contradictions:
+        return EditResult(errors=contradictions)
+    return EditResult(test=updated)
+
+
+def _copy_with_edits(
+    test: RegressionTest,
+    case_input: CaseInput,
+    expectations: list[Expectation],
+    *,
+    editor: str,
+) -> RegressionTest:
+    now = datetime.now(UTC)
+    updated = RegressionTest(
+        test_id=test.test_id,
+        failure_id=test.failure_id,
+        input=case_input,
+        provenance=test.provenance,
+        status=test.status,
+        fixtures=test.fixtures,
+        expectations=expectations,
+        reviewer=test.reviewer,
+        review_reason=test.review_reason,
+        reviewed_at=test.reviewed_at,
+        edited=True,
+        edited_by=editor,
+        created_at=test.created_at,
+        updated_at=now,
+    )
+    updated.warnings = recompute_warnings(updated)
+    return updated
+
+
+def _parse_input(raw: Any) -> tuple[CaseInput, list[str]]:
+    if raw is None:
+        return CaseInput(), ["input is required."]
+    if not isinstance(raw, dict):
+        return CaseInput(), [f"input must be a mapping, got {type(raw).__name__}."]
+
+    text = raw.get("text")
+    messages = raw.get("messages") or []
+    if text is not None and not isinstance(text, str):
+        return CaseInput(), ["input.text must be text."]
+    if not isinstance(messages, list):
+        return CaseInput(), ["input.messages must be a list."]
+
+    parsed: list[dict[str, str]] = []
+    for index, message in enumerate(messages):
+        if not isinstance(message, dict) or "role" not in message or "content" not in message:
+            return CaseInput(), [f"input.messages[{index}] needs a role and content."]
+        parsed.append({"role": str(message["role"]), "content": str(message["content"])})
+
+    if not (text or "").strip() and not parsed:
+        return CaseInput(), ["input needs non-empty text or at least one message."]
+    return CaseInput(text=text, messages=parsed), []
+
+
+def _parse_expectations(raw: Any) -> tuple[list[Expectation], list[str]]:
+    if raw is None or raw == []:
+        return [], ["A test needs at least one expectation."]
+    if not isinstance(raw, list):
+        return [], [f"expectations must be a list, got {type(raw).__name__}."]
+
+    expectations: list[Expectation] = []
+    errors: list[str] = []
+    for index, item in enumerate(raw):
+        if not isinstance(item, dict):
+            errors.append(f"expectations[{index}] must be a mapping.")
+            continue
+        kind = item.get("type")
+        try:
+            expectation_type = ExpectationType(str(kind))
+        except ValueError:
+            known = ", ".join(member.value for member in ExpectationType)
+            errors.append(f"expectations[{index}]: unknown type {kind!r}. Known types: {known}.")
+            continue
+
+        expectation = Expectation(
+            type=expectation_type,
+            value=item.get("value"),
+            tool=item.get("tool"),
+            path=item.get("path"),
+        )
+        problem = validate_expectation(expectation)
+        if problem is not None:
+            errors.append(f"expectations[{index}]: {problem}")
+            continue
+        expectations.append(expectation)
+
+    return expectations, errors

evalkeep/runner.py

@@ -0,0 +1,302 @@
+"""Delegating execution to Promptfoo, and reading the results back.
+
+Two things this module is careful about:
+
+* **The runner is invoked as an argument list, never through a shell.** Test
+  inputs, tool names and file paths all come from recorded traces. Building a
+  command string out of them would make a trace containing ``; rm -rf`` a
+  remote-code-execution bug, so ``subprocess.run`` is called with a list and
+  ``shell=False``, and nothing is ever passed through a shell.
+* **A test that never ran is not a test that failed.** Promptfoo distinguishes
+  an assertion failure from a provider error, and so does the import: an error
+  is recorded as :class:`~evalkeep.runs.Outcome.ERROR`, never as a failure.
+  Letting a crashed provider look like a regression is precisely the wrong
+  answer for a tool whose job is deciding whether a release got worse.
+"""
+
+from __future__ import annotations
+
+import json
+import platform
+import subprocess
+import sys
+import uuid
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from evalkeep.errors import CommandError
+from evalkeep.exporters.promptfoo import build_config
+from evalkeep.redaction import RedactionSummary, Redactor
+from evalkeep.regression import RegressionTest
+from evalkeep.runs import (
+    CaseResult,
+    CaseSummary,
+    ErrorKind,
+    EvaluationRun,
+    Outcome,
+    RunStatus,
+    suite_hash,
+    summarize,
+)
+from evalkeep.targets import Target, referenced_environment
+
+CONFIG_FILENAME = "promptfooconfig.yaml"
+RESULTS_FILENAME = "results.json"
+
+#: Promptfoo's own failure taxonomy, which the import preserves rather than
+#: flattening: 0 none, 1 assertion, 2 provider/execution error.
+_ASSERTION_FAILURE = 1
+_EXECUTION_ERROR = 2
+
+_TIMEOUT_MARKERS = ("timeout", "timed out", "etimedout", "esockettimedout")
+
+
+@dataclass
+class RunOutcome:
+    run: EvaluationRun
+    results: list[CaseResult] = field(default_factory=list)
+    #: Anything the runner said that a person should see.
+    messages: list[str] = field(default_factory=list)
+
+    @property
+    def counts(self) -> dict[Outcome, int]:
+        """Per-execution counts. With repetitions these exceed the test count."""
+        tally: dict[Outcome, int] = {}
+        for result in self.results:
+            tally[result.outcome] = tally.get(result.outcome, 0) + 1
+        return tally
+
+    @property
+    def summaries(self) -> dict[str, CaseSummary]:
+        """Per-case verdicts, which is what a repeated run is actually for."""
+        return summarize(self.results)
+
+    @property
+    def flaky(self) -> list[CaseSummary]:
+        return [s for s in self.summaries.values() if s.flaky]
+
+
+def write_suite(
+    tests: list[RegressionTest],
+    target: Target,
+    directory: Path,
+    *,
+    project_root: Path | None = None,
+) -> Path:
+    """Write a Promptfoo configuration for ``tests`` into ``directory``."""
+    directory.mkdir(parents=True, exist_ok=True)
+    config = build_config(tests, target, project_root=project_root, config_dir=directory)
+    path = directory / CONFIG_FILENAME
+    path.write_text(
+        yaml.safe_dump(config, sort_keys=False, default_flow_style=False, allow_unicode=True),
+        encoding="utf-8",
+    )
+    return path
+
+
+def execute(
+    tests: list[RegressionTest],
+    target: Target,
+    *,
+    directory: Path,
+    command: list[str],
+    timeout_seconds: int,
+    working_directory: Path,
+    redactor: Redactor | None = None,
+    repetitions: int = 1,
+) -> RunOutcome:
+    """Run the suite against ``target`` and import what came back."""
+    missing = [name for name, present in referenced_environment(target).items() if not present]
+    if missing:
+        raise CommandError(
+            f"Target {target.target_id!r} needs environment variables that are not "
+            f"set: {', '.join(sorted(missing))}.",
+            hint="Export them, or put them in a .env file that is not committed.",
+        )
+
+    config_path = write_suite(tests, target, directory, project_root=working_directory)
+    results_path = directory / RESULTS_FILENAME
+
+    run = EvaluationRun(
+        run_id=uuid.uuid4().hex,
+        target_id=target.target_id,
+        suite_hash=suite_hash([test.test_id for test in tests]),
+        tests=len(tests),
+        repetitions=repetitions,
+        environment=_environment(),
+        output_dir=str(directory),
+    )
+
+    argv = [
+        *command,
+        "eval",
+        "--config",
+        str(config_path),
+        "--output",
+        str(results_path),
+        # Caching would defeat the purpose: repeating a call and getting the
+        # cached answer back measures the cache, not the agent.
+        "--no-cache",
+    ]
+    if repetitions > 1:
+        argv += ["--repeat", str(repetitions)]
+    try:
+        # shell=False is the default and is relied upon: every element here can
+        # contain text that came out of a recorded trace.
+        completed = subprocess.run(
+            argv,
+            cwd=working_directory,
+            capture_output=True,
+            text=True,
+            timeout=timeout_seconds,
+            check=False,
+            shell=False,
+        )
+    except FileNotFoundError as exc:
+        raise CommandError(
+            f"Could not run {command[0]!r}: {exc}.",
+            hint="Promptfoo runs on Node. Install Node.js, or set runner.command.",
+        ) from exc
+    except subprocess.TimeoutExpired as exc:
+        run.status = RunStatus.FAILED
+        run.finished_at = datetime.now(UTC)
+        raise CommandError(
+            f"The runner did not finish within {timeout_seconds}s.",
+            hint="Raise runner.timeout_seconds, or run fewer tests with --limit.",
+        ) from exc
+
+    messages: list[str] = []
+    if not results_path.is_file():
+        raise CommandError(
+            "The runner produced no results file.",
+            hint=_tail(completed.stderr or completed.stdout),
+        )
+
+    results = import_results(results_path, redactor=redactor or Redactor())
+    run.finished_at = datetime.now(UTC)
+    run.runner = _runner_version(results_path)
+    # A non-zero exit is expected when tests fail; it only matters when nothing
+    # came back at all, which the missing-file check above already covers.
+    if completed.returncode != 0 and not results:
+        run.status = RunStatus.FAILED
+        messages.append(_tail(completed.stderr or completed.stdout))
+
+    return RunOutcome(run=run, results=results, messages=messages)
+
+
+def import_results(path: Path, *, redactor: Redactor | None = None) -> list[CaseResult]:
+    """Read a Promptfoo results file into per-test outcomes."""
+    redactor = redactor or Redactor()
+    try:
+        payload: Any = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise CommandError(f"Could not read the runner's results at {path}: {exc}") from exc
+
+    records = (payload.get("results") or {}).get("results") or []
+
+    # The runner returns one row per execution, in order, with repeated runs of
+    # the same case sharing a test ID. Numbering them here is what lets a case
+    # be judged on all of its attempts rather than its last one.
+    seen: dict[str, int] = {}
+    results: list[CaseResult] = []
+    for record in records:
+        test_id = _test_id(record)
+        if not test_id:
+            continue
+        repetition = seen.get(test_id, 0)
+        seen[test_id] = repetition + 1
+        results.append(_result(record, redactor, repetition))
+    return results
+
+
+def _result(record: dict[str, Any], redactor: Redactor, repetition: int = 0) -> CaseResult:
+    test_id = _test_id(record)
+    failure_reason = record.get("failureReason") or 0
+    error = record.get("error") or None
+
+    if record.get("success"):
+        outcome, error_kind = Outcome.PASS, None
+    elif failure_reason == _EXECUTION_ERROR:
+        outcome = Outcome.ERROR
+        error_kind = (
+            ErrorKind.TIMEOUT
+            if any(marker in (error or "").lower() for marker in _TIMEOUT_MARKERS)
+            else ErrorKind.EXECUTION_ERROR
+        )
+    elif failure_reason == _ASSERTION_FAILURE:
+        outcome, error_kind = Outcome.FAIL, None
+    else:
+        # A failure Promptfoo did not classify is not assumed to be the agent's
+        # fault; it is reported as an error so it cannot masquerade as one.
+        outcome = Outcome.ERROR
+        error_kind = ErrorKind.EXECUTION_ERROR
+
+    summary = RedactionSummary()
+    return CaseResult(
+        test_id=test_id,
+        outcome=outcome,
+        error_kind=error_kind,
+        error=redactor.redact_text(error, summary) if error else None,
+        latency_ms=record.get("latencyMs"),
+        observation=redactor.redact_text(_observation(record), summary) or None,
+        failed_assertions=[
+            redactor.redact_text(reason, summary) for reason in _failed_assertions(record)
+        ],
+        repetition=repetition,
+    )
+
+
+def _test_id(record: dict[str, Any]) -> str:
+    case = record.get("testCase") or {}
+    metadata = case.get("metadata") or {}
+    return str(metadata.get("test_id") or case.get("description") or "")
+
+
+def _observation(record: dict[str, Any]) -> str:
+    response = record.get("response") or {}
+    output = response.get("output")
+    if output is None:
+        return ""
+    if isinstance(output, str):
+        return output
+    return json.dumps(output, sort_keys=True, default=str)
+
+
+def _failed_assertions(record: dict[str, Any]) -> list[str]:
+    grading = record.get("gradingResult") or {}
+    components = grading.get("componentResults") or []
+    reasons = [
+        str(component.get("reason", "")).strip()
+        for component in components
+        if not component.get("pass", True)
+    ]
+    if reasons:
+        return reasons
+    reason = str(grading.get("reason", "")).strip()
+    return [reason] if reason and not grading.get("pass", True) else []
+
+
+def _runner_version(results_path: Path) -> str | None:
+    try:
+        payload = json.loads(results_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):  # pragma: no cover - already parsed once
+        return None
+    version = (payload.get("results") or {}).get("version")
+    return f"promptfoo:{version}" if version else None
+
+
+def _environment() -> dict[str, Any]:
+    """What the run happened on, so a surprising result can be placed."""
+    return {
+        "python": sys.version.split()[0],
+        "platform": platform.platform(),
+        "machine": platform.machine(),
+    }
+
+
+def _tail(text: str, lines: int = 12) -> str:
+    return "\n".join((text or "").strip().splitlines()[-lines:])