evalkeep 0.1.0__py3-none-any.whl

evalkeep/commands/review_cmd.py

@@ -0,0 +1,141 @@
+"""``evalkeep review`` and the non-interactive decision commands.
+
+The interactive loop lives in the CLI; everything it can do is also reachable
+one decision at a time, so a script or a CI job can record the same decisions
+without a terminal.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from evalkeep.analysis import FailureAnalysis
+from evalkeep.commands.dataset_cmd import show_test
+from evalkeep.commands.detect_cmd import default_reviewer
+from evalkeep.config import Project
+from evalkeep.errors import CommandError
+from evalkeep.failures import Failure
+from evalkeep.regression import RegressionTest, ReviewStatus
+from evalkeep.review import ReviewError, apply_edits, approve, reject, render_editable
+from evalkeep.storage import StoredTrace, TraceStore
+
+
+@dataclass(frozen=True)
+class ReviewItem:
+    """Everything a reviewer needs on screen to decide: guide 8H's first rule."""
+
+    test: RegressionTest
+    trace: StoredTrace
+    failure: Failure
+    analysis: FailureAnalysis | None
+
+
+def pending_reviews(*, project_root: Path = Path(), limit: int | None = None) -> list[ReviewItem]:
+    """Drafts awaiting a decision, with their source interaction and analysis."""
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        drafts = store.tests.list(status=ReviewStatus.DRAFT, limit=limit or 1000)
+        return [item for test in drafts if (item := _assemble(store, test)) is not None]
+
+
+def review_item(identifier: str, *, project_root: Path = Path()) -> ReviewItem:
+    """One test, with its context, whatever its status."""
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        test = _resolve(store, identifier)
+        item = _assemble(store, test)
+        if item is None:  # pragma: no cover - foreign keys prevent this
+            raise CommandError(f"Test {test.test_id!r} has lost its source trace.")
+        return item
+
+
+def approve_test(
+    identifier: str,
+    *,
+    project_root: Path = Path(),
+    reviewer: str | None = None,
+    reason: str | None = None,
+) -> RegressionTest:
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        test = _resolve(store, identifier)
+        try:
+            approved = approve(test, reviewer=reviewer or default_reviewer(), reason=reason)
+        except ReviewError as exc:
+            raise CommandError(
+                str(exc),
+                hint="Fix it with 'evalkeep dataset edit <id>' and approve again.",
+            ) from exc
+        store.tests.save(approved)
+        return approved
+
+
+def reject_test(
+    identifier: str,
+    *,
+    project_root: Path = Path(),
+    reviewer: str | None = None,
+    reason: str | None = None,
+) -> RegressionTest:
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        test = _resolve(store, identifier)
+        rejected = reject(test, reviewer=reviewer or default_reviewer(), reason=reason)
+        store.tests.save(rejected)
+        return rejected
+
+
+def editable_document(identifier: str, *, project_root: Path = Path()) -> str:
+    """The YAML a reviewer edits for one test."""
+    return render_editable(show_test(identifier, project_root=project_root))
+
+
+def edit_test(
+    identifier: str,
+    document: str,
+    *,
+    project_root: Path = Path(),
+    editor: str | None = None,
+) -> RegressionTest:
+    """Apply an edited document. Nothing is stored unless it is valid."""
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        test = _resolve(store, identifier)
+        result = apply_edits(test, document, editor=editor or default_reviewer())
+        if result.test is None:
+            raise CommandError(
+                "The edit was not applied:\n  - " + "\n  - ".join(result.errors),
+                hint="The stored draft is unchanged.",
+            )
+        store.tests.save(result.test)
+        return result.test
+
+
+def _assemble(store: TraceStore, test: RegressionTest) -> ReviewItem | None:
+    failure = store.failures.get(test.failure_id)
+    stored = store.get(test.provenance.trace_id)
+    if failure is None or stored is None:  # pragma: no cover - foreign keys prevent this
+        return None
+    return ReviewItem(
+        test=test,
+        trace=stored,
+        failure=failure,
+        analysis=store.failures.get_analysis(test.failure_id),
+    )
+
+
+def _resolve(store: TraceStore, identifier: str) -> RegressionTest:
+    """Accept a test ID, a failure ID, or the trace the test came from."""
+    cleaned = identifier.strip()
+    test = store.tests.get(cleaned) or store.tests.get_by_failure(cleaned)
+    if test is None:
+        failure = store.failures.get_by_trace(cleaned)
+        if failure is not None:
+            test = store.tests.get_by_failure(failure.failure_id)
+    if test is None:
+        raise CommandError(
+            f"No regression test matching {cleaned!r}.",
+            hint="Run 'evalkeep dataset list' to see what exists.",
+        )
+    return test

evalkeep/commands/run_cmd.py

@@ -0,0 +1,131 @@
+"""``evalkeep export`` and ``evalkeep run`` -- hand the suite to the runner."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+from evalkeep.config import Project
+from evalkeep.errors import CommandError
+from evalkeep.exporters import ExportFormat, build_config, replay_warnings, to_jsonl
+from evalkeep.redaction import Redactor
+from evalkeep.regression import RegressionTest, ReviewStatus
+from evalkeep.runner import RunOutcome, execute
+from evalkeep.storage import TraceStore
+from evalkeep.targets import Target, get_target
+
+NOTHING_APPROVED = "No approved tests to export."
+NOTHING_APPROVED_HINT = "Only approved tests leave the database. Run 'evalkeep review' first."
+
+
+@dataclass(frozen=True)
+class ExportResult:
+    format: ExportFormat
+    path: Path
+    tests: int
+    target_id: str | None = None
+    warnings: tuple[str, ...] = ()
+
+
+def approved_tests(*, project_root: Path = Path()) -> list[RegressionTest]:
+    """The suite: every approved test, and nothing else."""
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        return store.tests.list(status=ReviewStatus.APPROVED, limit=10_000)
+
+
+def export_suite(
+    *,
+    project_root: Path = Path(),
+    export_format: ExportFormat = ExportFormat.PROMPTFOO,
+    target_id: str | None = None,
+    out: Path | None = None,
+) -> ExportResult:
+    """Write the approved suite in the requested format."""
+    project = Project.load(project_root.expanduser().resolve())
+    tests = approved_tests(project_root=project_root)
+    if not tests:
+        raise CommandError(NOTHING_APPROVED, hint=NOTHING_APPROVED_HINT)
+
+    directory = (out or project.subdir("exports")).expanduser()
+    directory.mkdir(parents=True, exist_ok=True)
+
+    if export_format is ExportFormat.JSONL:
+        path = directory / "tests.jsonl"
+        path.write_text(to_jsonl(tests), encoding="utf-8")
+        return ExportResult(format=export_format, path=path, tests=len(tests))
+
+    target = _target(project, target_id)
+    path = directory / f"promptfooconfig.{target.target_id}.yaml"
+    path.write_text(
+        yaml.safe_dump(build_config(tests, target), sort_keys=False, default_flow_style=False),
+        encoding="utf-8",
+    )
+    return ExportResult(
+        format=export_format,
+        path=path,
+        tests=len(tests),
+        target_id=target.target_id,
+        warnings=tuple(replay_warnings(tests, target)),
+    )
+
+
+def run_suite(
+    *,
+    project_root: Path = Path(),
+    target_id: str,
+    limit: int | None = None,
+    repetitions: int = 1,
+) -> RunOutcome:
+    """Delegate execution of the approved suite to the configured runner."""
+    project = Project.load(project_root.expanduser().resolve())
+    target = get_target(project.root, target_id)
+    target.validate_shape()
+
+    tests = approved_tests(project_root=project_root)
+    if not tests:
+        raise CommandError(NOTHING_APPROVED, hint="Run 'evalkeep review' first.")
+    if limit is not None:
+        tests = tests[:limit]
+
+    warnings = replay_warnings(tests, target)
+    outcome = execute(
+        tests,
+        target,
+        directory=project.subdir("runs") / f"{target.target_id}-pending",
+        command=list(project.config.runner.command),
+        timeout_seconds=project.config.runner.timeout_seconds,
+        repetitions=repetitions,
+        # Relative paths in a target are relative to the project, not to
+        # wherever the command happened to be typed.
+        working_directory=project.root,
+        redactor=Redactor(project.config.redaction),
+    )
+
+    # Name the directory after the run only once the run has an identity.
+    final = project.subdir("runs") / outcome.run.run_id
+    pending = Path(outcome.run.output_dir or "")
+    if pending.is_dir() and not final.exists():
+        pending.rename(final)
+        outcome.run.output_dir = str(final)
+
+    outcome.messages[:0] = warnings
+    with TraceStore.open(project.database_path) as store:
+        store.runs.save(outcome.run, outcome.results)
+    return outcome
+
+
+def _target(project: Project, target_id: str | None) -> Target:
+    if target_id is not None:
+        return get_target(project.root, target_id)
+    from evalkeep.targets import load_targets
+
+    targets = load_targets(project.root)
+    if len(targets.targets) == 1:
+        return next(iter(targets.targets.values()))
+    raise CommandError(
+        "Which target should this be exported for?",
+        hint="Pass --target, or add one with 'evalkeep targets add'.",
+    )

evalkeep/commands/target_cmd.py

@@ -0,0 +1,109 @@
+"""``evalkeep targets`` -- configure the agents under test."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from evalkeep.config import Project
+from evalkeep.errors import CommandError
+from evalkeep.targets import (
+    Extraction,
+    Target,
+    TargetKind,
+    find_secrets,
+    load_targets,
+    referenced_environment,
+    save_targets,
+)
+
+SECRET_HINT = (
+    "Reference secrets as ${ENV_VAR}; targets.yaml is committed, so a literal "
+    "credential here would be a leak."
+)
+
+
+def add_target(
+    target_id: str,
+    kind: TargetKind,
+    *,
+    project_root: Path = Path(),
+    description: str | None = None,
+    url: str | None = None,
+    method: str = "POST",
+    headers: dict[str, str] | None = None,
+    body: dict[str, object] | None = None,
+    path: str | None = None,
+    function: str | None = None,
+    provider: str | None = None,
+    output_path: str | None = None,
+    tool_calls_path: str | None = None,
+    replace: bool = False,
+) -> Target:
+    """Record a target, refusing anything that carries a literal credential."""
+    project = Project.load(project_root.expanduser().resolve())
+    file = load_targets(project.root)
+
+    cleaned = target_id.strip()
+    if not cleaned:
+        raise CommandError("A target needs a name.")
+    if cleaned in file.targets and not replace:
+        raise CommandError(
+            f"A target named {cleaned!r} already exists.",
+            hint="Pass --replace to overwrite it.",
+        )
+
+    extract = Extraction()
+    if output_path is not None:
+        extract.output = output_path
+    if tool_calls_path is not None:
+        extract.tool_calls = tool_calls_path
+
+    target = Target(
+        target_id=cleaned,
+        kind=kind,
+        description=description,
+        url=url,
+        method=method,
+        headers=headers or {},
+        body=body or {},
+        extract=extract,
+        path=path,
+        function=function,
+        provider=provider,
+    )
+    target.validate_shape()
+
+    # The check that makes targets.yaml safe to commit.
+    problems = find_secrets(target)
+    if problems:
+        raise CommandError(
+            "This target contains what looks like a credential:\n  - " + "\n  - ".join(problems),
+            hint=SECRET_HINT,
+        )
+
+    file.targets[cleaned] = target
+    save_targets(project.root, file)
+    return target
+
+
+def list_targets(*, project_root: Path = Path()) -> list[Target]:
+    project = Project.load(project_root.expanduser().resolve())
+    return sorted(load_targets(project.root).targets.values(), key=lambda t: t.target_id)
+
+
+def show_target(target_id: str, *, project_root: Path = Path()) -> tuple[Target, dict[str, bool]]:
+    """A target and the environment variables it depends on."""
+    from evalkeep.targets import get_target
+
+    project = Project.load(project_root.expanduser().resolve())
+    target = get_target(project.root, target_id)
+    return target, referenced_environment(target)
+
+
+def remove_target(target_id: str, *, project_root: Path = Path()) -> None:
+    project = Project.load(project_root.expanduser().resolve())
+    file = load_targets(project.root)
+    if target_id.strip() not in file.targets:
+        raise CommandError(f"No target named {target_id.strip()!r}.")
+    del file.targets[target_id.strip()]
+    save_targets(project.root, file)

evalkeep/commands/trace_cmd.py

@@ -0,0 +1,58 @@
+"""``evalkeep trace`` -- inspect stored traces.
+
+Safe by construction: the database only ever holds redacted traces, so there is
+no unredacted view to expose here.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from evalkeep.config import Project
+from evalkeep.errors import CommandError
+from evalkeep.storage import StoredTrace, TraceStore, TraceSummary
+
+
+@dataclass(frozen=True)
+class TraceListing:
+    summaries: list[TraceSummary]
+    total: int
+    offset: int
+
+
+def show_trace(trace_id: str, *, project_root: Path = Path()) -> StoredTrace:
+    """Load one stored trace, or explain that it is not there."""
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        stored = next(
+            (
+                found
+                for candidate in project.identify(trace_id)
+                if (found := store.get(candidate)) is not None
+            ),
+            None,
+        )
+        if stored is None:
+            raise CommandError(
+                f"No stored trace with ID {trace_id.strip()!r}.",
+                hint="Run 'evalkeep trace list' to see what has been ingested.",
+            )
+        return stored
+
+
+def list_traces(
+    *,
+    project_root: Path = Path(),
+    limit: int = 50,
+    offset: int = 0,
+    status: str | None = None,
+) -> TraceListing:
+    """Page through stored traces, newest ingest last."""
+    project = Project.load(project_root.expanduser().resolve())
+    with TraceStore.open(project.database_path) as store:
+        return TraceListing(
+            summaries=store.list(limit=limit, offset=offset, status=status),
+            total=store.count(status=status),
+            offset=offset,
+        )