evalctl 0.3.0__tar.gz → 0.4.2__tar.gz

{evalctl-0.3.0 → evalctl-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evalctl
-Version: 0.3.0
+Version: 0.4.2
 Summary: Local-first evals for agents, not just prompts.
 Author: evalctl contributors
 License:                                  Apache License
@@ -217,6 +217,8 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: coverage>=7; extra == "dev"
 Dynamic: license-file
 
 # evalctl
@@ -232,11 +234,13 @@ durable artifacts. It scores what agents actually do — files written, diffs
 produced, commands run — on your own machine, with no gateway, dashboard, or
 SaaS account.
 
-v0.3 writes durable run metadata, supports crash resume, adds local run-state
-inspection, and can optionally delegate runner execution to
+v0.4 writes durable run metadata, supports crash resume, adds local run-state
+inspection, diagnoses runtime health, produces side-effect-free execution plans,
+and can optionally delegate runner execution to
 [spoolctl](https://github.com/Ozhiaki/spoolctl). The standalone synchronous path
 remains complete and requires no external service. [inferctl](https://inferctl.dev)
-route capture remains deferred.
+preflight provenance can be captured before runner execution without changing
+report scoring.
 
 ## About
 
@@ -259,17 +263,17 @@ evalctl is agent-shaped.
 | --- | --- | --- |
 | Unit under test | prompt → completion | agent run → files, diffs, command logs, artifacts |
 | Scoring surface | text of a response | resulting workspace: git diff, expected/forbidden file changes, exit codes, plus text |
-| Execution | in-process, synchronous | v0.3 synchronous by default; `run --resume` resumes crashed runs; optional `--queue spoolctl` delegates runner execution |
-| Model context | provider API keys | [inferctl](https://inferctl.dev) route/preflight provenance is deferred |
+| Execution | in-process, synchronous | v0.4 synchronous by default; `plan` previews actions, `doctor` diagnoses state, `run --resume` resumes crashed runs; optional `--queue spoolctl` delegates runner execution |
+| Model context | provider API keys | Optional [inferctl](https://inferctl.dev) preflight provenance through `run --inferctl-task`; route capture remains deferred |
 
 ## Status
 
-Python pre-release. v0.3 provides scaffold, validate, bounded parallel run
-execution, durable run metadata, crash resume, local jobs inspection, optional
-spoolctl queueing, status, report, deterministic local scorers, CLI authoring
-verbs, execution replay for failed cases, command scorers, truthful
-warnings/errors, real schema output, and artifact replay from a copied run
-directory.
+Python pre-release. v0.4 provides scaffold, validate, doctor, plan, bounded
+parallel run execution, durable run metadata, crash resume, bounded local jobs
+inspection, optional spoolctl queueing, optional inferctl preflight provenance,
+status, report, deterministic local scorers, CLI authoring verbs, execution
+replay for failed cases, command scorers, truthful warnings/errors, real schema
+output, and artifact replay from a copied run directory.
 `contract_version` remains `1`.
 
 ## Install
@@ -300,8 +304,11 @@ evalctl suite add demo --runner-argv "python3 $EVALCTL_WORKSPACE/r.py" --json
 evalctl case add demo --task "do X" --workspace fixtures/x --expect-json '{"exact":"ok"}' --json
 evalctl scorer add demo --name exact --required --json
 evalctl run demo --json
+evalctl doctor --json
+evalctl plan demo --json
+evalctl run demo --inferctl-task code --json
 evalctl run --resume <run-id> --json
-evalctl jobs list --json
+evalctl jobs list --limit 50 --json
 evalctl run demo --queue spoolctl --slots 4 --json
 evalctl replay --failed <run-id> --json
 evalctl report <run-id> --format json
@@ -322,23 +329,55 @@ reservation is reclaimed by explicit `--resume`. `jobs list|get|prune` inspects
 completed, running, stale, and orphaned local run state and safely prunes only
 with explicit confirmation.
 
+`jobs list` is bounded by default. Use `--limit` and `--cursor` to page through
+large run directories; the JSON envelope includes pagination metadata and a
+paste-ready next-page command when more rows are available.
+
 Durability sidecars are operational state. Reports and artifact replay do not
 require `run.json`, `.reservation.json`, `.spoolctl.db`, `state.json`, or
 `job.json`; `report_hash` stays based on the report projection. `SOURCE_DATE_EPOCH`
 controls `created_ts` for deterministic manifest parity.
 
+## Doctor And Plan
+
+`evalctl doctor --json` reports runtime, suite root, runs root, reservations,
+spoolctl, inferctl, and runner-safety state without failing just because a
+component is degraded. Use `--component NAME` to scope diagnostics and `--fast`
+for PATH-only optional integration checks.
+
+`evalctl plan <suite> --json` resolves the case set without creating run
+directories, enqueueing jobs, executing runners, scoring, or writing inferctl
+artifacts. The plan includes run-id strategy, execution mode, independent-case
+dependency metadata, parallel tracks, per-case actions, warnings, and
+paste-ready follow-up commands. `--resume`, `--queue spoolctl`, `--slots`, and
+`--inferctl-task` mirror the run surface for planning.
+
 ## Optional Spoolctl Queue
 
 `evalctl run <suite> --queue spoolctl --json` delegates only runner execution to
-spoolctl (`>= 0.4.1`). Evalctl still prepares workspaces, normalizes stdout and
+spoolctl (`>= 0.4.11`, speaking contract `>= 2`). A spoolctl reporting a newer
+contract is accepted. Evalctl still prepares workspaces, normalizes stdout and
 stderr, captures workspace diffs, scores cases, and writes terminal markers. If
 spoolctl is absent or incompatible, queued runs fail explicitly; non-queued runs
 do not need spoolctl.
 
-The queue database is per-run at `.spoolctl.db`; v0.3 starts one ephemeral
+The queue database is per-run at `.spoolctl.db`; v0.4 starts one ephemeral
 `spoolctl work --drain` worker per queued run. General externally managed worker
 fleets are not part of this release.
 
+## Inferctl Preflight Provenance
+
+`evalctl run <suite> --inferctl-task TASK --json` probes inferctl once per run
+and, when compatible preflight support is available, writes per-case
+`inferctl-preflight.json` and `inferctl-provenance.json` before runner execution.
+Queued spoolctl runs capture the same artifacts before enqueue. Absence,
+incompatibility, parse failures, timeouts, and policy/readiness blocks are
+warnings; the runner still executes and scoring proceeds.
+
+The v0.4 capture mode is `preflight` only. `inferctl route` is not called, and
+report projection is unchanged, so `report_hash` remains comparable to an
+equivalent run without inferctl.
+
 ## Authoring
 
 `suite add`, `case add`, and `scorer add` let agents build a suite without

{evalctl-0.3.0 → evalctl-0.4.2}/README.md

@@ -11,11 +11,13 @@ durable artifacts. It scores what agents actually do — files written, diffs
 produced, commands run — on your own machine, with no gateway, dashboard, or
 SaaS account.
 
-v0.3 writes durable run metadata, supports crash resume, adds local run-state
-inspection, and can optionally delegate runner execution to
+v0.4 writes durable run metadata, supports crash resume, adds local run-state
+inspection, diagnoses runtime health, produces side-effect-free execution plans,
+and can optionally delegate runner execution to
 [spoolctl](https://github.com/Ozhiaki/spoolctl). The standalone synchronous path
 remains complete and requires no external service. [inferctl](https://inferctl.dev)
-route capture remains deferred.
+preflight provenance can be captured before runner execution without changing
+report scoring.
 
 ## About
 
@@ -38,17 +40,17 @@ evalctl is agent-shaped.
 | --- | --- | --- |
 | Unit under test | prompt → completion | agent run → files, diffs, command logs, artifacts |
 | Scoring surface | text of a response | resulting workspace: git diff, expected/forbidden file changes, exit codes, plus text |
-| Execution | in-process, synchronous | v0.3 synchronous by default; `run --resume` resumes crashed runs; optional `--queue spoolctl` delegates runner execution |
-| Model context | provider API keys | [inferctl](https://inferctl.dev) route/preflight provenance is deferred |
+| Execution | in-process, synchronous | v0.4 synchronous by default; `plan` previews actions, `doctor` diagnoses state, `run --resume` resumes crashed runs; optional `--queue spoolctl` delegates runner execution |
+| Model context | provider API keys | Optional [inferctl](https://inferctl.dev) preflight provenance through `run --inferctl-task`; route capture remains deferred |
 
 ## Status
 
-Python pre-release. v0.3 provides scaffold, validate, bounded parallel run
-execution, durable run metadata, crash resume, local jobs inspection, optional
-spoolctl queueing, status, report, deterministic local scorers, CLI authoring
-verbs, execution replay for failed cases, command scorers, truthful
-warnings/errors, real schema output, and artifact replay from a copied run
-directory.
+Python pre-release. v0.4 provides scaffold, validate, doctor, plan, bounded
+parallel run execution, durable run metadata, crash resume, bounded local jobs
+inspection, optional spoolctl queueing, optional inferctl preflight provenance,
+status, report, deterministic local scorers, CLI authoring verbs, execution
+replay for failed cases, command scorers, truthful warnings/errors, real schema
+output, and artifact replay from a copied run directory.
 `contract_version` remains `1`.
 
 ## Install
@@ -79,8 +81,11 @@ evalctl suite add demo --runner-argv "python3 $EVALCTL_WORKSPACE/r.py" --json
 evalctl case add demo --task "do X" --workspace fixtures/x --expect-json '{"exact":"ok"}' --json
 evalctl scorer add demo --name exact --required --json
 evalctl run demo --json
+evalctl doctor --json
+evalctl plan demo --json
+evalctl run demo --inferctl-task code --json
 evalctl run --resume <run-id> --json
-evalctl jobs list --json
+evalctl jobs list --limit 50 --json
 evalctl run demo --queue spoolctl --slots 4 --json
 evalctl replay --failed <run-id> --json
 evalctl report <run-id> --format json
@@ -101,23 +106,55 @@ reservation is reclaimed by explicit `--resume`. `jobs list|get|prune` inspects
 completed, running, stale, and orphaned local run state and safely prunes only
 with explicit confirmation.
 
+`jobs list` is bounded by default. Use `--limit` and `--cursor` to page through
+large run directories; the JSON envelope includes pagination metadata and a
+paste-ready next-page command when more rows are available.
+
 Durability sidecars are operational state. Reports and artifact replay do not
 require `run.json`, `.reservation.json`, `.spoolctl.db`, `state.json`, or
 `job.json`; `report_hash` stays based on the report projection. `SOURCE_DATE_EPOCH`
 controls `created_ts` for deterministic manifest parity.
 
+## Doctor And Plan
+
+`evalctl doctor --json` reports runtime, suite root, runs root, reservations,
+spoolctl, inferctl, and runner-safety state without failing just because a
+component is degraded. Use `--component NAME` to scope diagnostics and `--fast`
+for PATH-only optional integration checks.
+
+`evalctl plan <suite> --json` resolves the case set without creating run
+directories, enqueueing jobs, executing runners, scoring, or writing inferctl
+artifacts. The plan includes run-id strategy, execution mode, independent-case
+dependency metadata, parallel tracks, per-case actions, warnings, and
+paste-ready follow-up commands. `--resume`, `--queue spoolctl`, `--slots`, and
+`--inferctl-task` mirror the run surface for planning.
+
 ## Optional Spoolctl Queue
 
 `evalctl run <suite> --queue spoolctl --json` delegates only runner execution to
-spoolctl (`>= 0.4.1`). Evalctl still prepares workspaces, normalizes stdout and
+spoolctl (`>= 0.4.11`, speaking contract `>= 2`). A spoolctl reporting a newer
+contract is accepted. Evalctl still prepares workspaces, normalizes stdout and
 stderr, captures workspace diffs, scores cases, and writes terminal markers. If
 spoolctl is absent or incompatible, queued runs fail explicitly; non-queued runs
 do not need spoolctl.
 
-The queue database is per-run at `.spoolctl.db`; v0.3 starts one ephemeral
+The queue database is per-run at `.spoolctl.db`; v0.4 starts one ephemeral
 `spoolctl work --drain` worker per queued run. General externally managed worker
 fleets are not part of this release.
 
+## Inferctl Preflight Provenance
+
+`evalctl run <suite> --inferctl-task TASK --json` probes inferctl once per run
+and, when compatible preflight support is available, writes per-case
+`inferctl-preflight.json` and `inferctl-provenance.json` before runner execution.
+Queued spoolctl runs capture the same artifacts before enqueue. Absence,
+incompatibility, parse failures, timeouts, and policy/readiness blocks are
+warnings; the runner still executes and scoring proceeds.
+
+The v0.4 capture mode is `preflight` only. `inferctl route` is not called, and
+report projection is unchanged, so `report_hash` remains comparable to an
+equivalent run without inferctl.
+
 ## Authoring
 
 `suite add`, `case add`, and `scorer add` let agents build a suite without

{evalctl-0.3.0 → evalctl-0.4.2}/evalctl/__init__.py

@@ -1,3 +1,3 @@
 """evalctl package."""
 
-__version__ = "0.3.0"
+__version__ = "0.4.2"

evalctl-0.4.2/evalctl/artifacts.py

@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+import stat
+import tempfile
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+from .static_contract import EvalctlError, sha256_bytes, sha256_text, stable_json
+
+
+def read_json(path: Path) -> Any:
+    try:
+        return json.loads(path.read_text())
+    except FileNotFoundError:
+        raise EvalctlError("E_SUITE_NOT_FOUND", f"missing file: {path}", f"create {path} or run evalctl init", 1)
+    except json.JSONDecodeError as exc:
+        raise EvalctlError("E_SCHEMA_VIOLATION", f"invalid JSON in {path}: {exc.msg}", f"fix {path}:{exc.lineno}", 1)
+
+
+def load_cases(cases_path: Path) -> list[dict[str, Any]]:
+    cases: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    try:
+        lines = cases_path.read_text().splitlines()
+    except FileNotFoundError:
+        raise EvalctlError("E_SUITE_NOT_FOUND", f"missing file: {cases_path}", f"create {cases_path}", 1)
+    for line_no, line in enumerate(lines, 1):
+        if not line.strip():
+            continue
+        try:
+            case = json.loads(line)
+        except json.JSONDecodeError as exc:
+            raise EvalctlError("E_CASE_INVALID", f"invalid JSONL at {cases_path}:{line_no}: {exc.msg}", f"fix line {line_no}", 1)
+        case_id = case.get("id") or sha256_text(stable_json(case))[7:19]
+        if case_id in seen:
+            raise EvalctlError("E_CASE_INVALID", f"duplicate case id: {case_id}", "choose unique case ids", 1)
+        seen.add(case_id)
+        case["id"] = case_id
+        cases.append(case)
+    return cases
+
+
+def normalize_rel(path: Path, root: Path) -> str:
+    rel = path.relative_to(root)
+    parts = rel.parts
+    posix = PurePosixPath(*parts).as_posix()
+    if posix in ("", ".") or ".." in parts:
+        raise ValueError("invalid relative path")
+    posix.encode("utf-8", "strict")
+    return posix
+
+
+def display_path_name(path: Path) -> str:
+    return os.fsencode(path.name).decode("utf-8", "replace")
+
+
+def manifest(root: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+    warnings: list[dict[str, Any]] = []
+    entries: list[dict[str, Any]] = []
+    for path in sorted(root.rglob("*"), key=os.fsencode):
+        try:
+            rel = normalize_rel(path, root)
+            st = path.lstat()
+            if stat.S_ISDIR(st.st_mode):
+                entries.append({"path": rel, "kind": "directory"})
+            elif stat.S_ISLNK(st.st_mode):
+                target = os.readlink(path)
+                entries.append({"path": rel, "kind": "symlink", "target": target, "sha256": sha256_bytes(target.encode("utf-8")), "broken": not path.exists()})
+            elif stat.S_ISREG(st.st_mode):
+                entries.append({"path": rel, "kind": "file", "sha256": sha256_bytes(path.read_bytes()), "size": st.st_size})
+            else:
+                subtype = "fifo" if stat.S_ISFIFO(st.st_mode) else "socket" if stat.S_ISSOCK(st.st_mode) else "device" if stat.S_ISCHR(st.st_mode) or stat.S_ISBLK(st.st_mode) else "unknown"
+                entries.append({"path": rel, "kind": "other", "subtype": subtype})
+        except Exception as exc:
+            warnings.append({"code": "W_PATH_UNREADABLE", "message": f"could not read path {display_path_name(path)}: {exc}"})
+    return {"root": ".", "entries": entries}, warnings
+
+
+def diff_manifests(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]:
+    b = {e["path"]: e for e in before["entries"]}
+    a = {e["path"]: e for e in after["entries"]}
+    changed = []
+    for path in sorted(set(b) | set(a)):
+        if path not in b:
+            changed.append({"path": path, "status": "added", "kind": a[path]["kind"], "sha256_before": None, "sha256_after": a[path].get("sha256")})
+        elif path not in a:
+            changed.append({"path": path, "status": "deleted", "kind": b[path]["kind"], "sha256_before": b[path].get("sha256"), "sha256_after": None})
+        elif b[path].get("sha256") != a[path].get("sha256") or b[path].get("kind") != a[path].get("kind"):
+            changed.append({"path": path, "status": "modified", "kind": a[path]["kind"], "sha256_before": b[path].get("sha256"), "sha256_after": a[path].get("sha256")})
+    return {"changed_paths": changed}
+
+
+def apply_redaction(text: str, patterns: list[str], values: list[str]) -> tuple[str, bool]:
+    changed = False
+    for value in values:
+        if value and value in text:
+            text = text.replace(value, "[REDACTED]")
+            changed = True
+    for pattern in patterns:
+        text2 = re.sub(pattern, "[REDACTED]", text)
+        changed = changed or text2 != text
+        text = text2
+    return text, changed
+
+
+def _atomic_write(path: Path, text: str, *, _writer: Any | None = None) -> None:
+    writer = _writer or (lambda tmp_path, value: tmp_path.write_text(value))
+    tmp_name = None
+    with tempfile.NamedTemporaryFile("w", dir=path.parent, prefix=f".{path.name}.", suffix=".tmp", delete=False) as tmp:
+        tmp_name = tmp.name
+    tmp_path = Path(tmp_name)
+    try:
+        writer(tmp_path, text)
+        os.replace(tmp_path, path)
+    finally:
+        if tmp_path.exists():
+            tmp_path.unlink()
+
+
+def write_json(path: Path, data: Any) -> None:
+    _atomic_write(path, json.dumps(data, indent=2, sort_keys=True) + "\n")
+
+
+def render_text_diff(diff: dict[str, Any]) -> str:
+    return "\n".join(f"{p['status']}\t{p['path']}" for p in diff["changed_paths"]) + ("\n" if diff["changed_paths"] else "")

evalctl-0.4.2/evalctl/cli.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import sys
+import time
+
+from .commands import dispatch, report_format_json_requested, wants_json
+from .static_contract import EvalctlError, envelope, stable_json
+
+
+def print_error(err: EvalctlError, *, json_mode: bool, started: float | None = None) -> int:
+    print(err.error["message"], file=sys.stderr)
+    if err.error.get("corrected_command"):
+        print(f"Did you mean: {err.error['corrected_command']}", file=sys.stderr)
+    elif err.error.get("did_you_mean"):
+        print(f"Did you mean: {err.error['did_you_mean']}", file=sys.stderr)
+    if json_mode:
+        print(stable_json(envelope(None, ok=False, errors=[err.error], started=started)))
+    return err.exit_code
+
+
+def main(argv: list[str] | None = None) -> int:
+    started = time.time()
+    argv = list(sys.argv[1:] if argv is None else argv)
+    json_mode = wants_json(argv)
+    try:
+        return dispatch(argv, json_mode, started)
+    except EvalctlError as exc:
+        return print_error(exc, json_mode=json_mode or report_format_json_requested(argv), started=started)
+    except KeyboardInterrupt:
+        return print_error(EvalctlError("E_RUNNER_FAILED", "interrupted", "retry the command", 3), json_mode=json_mode or report_format_json_requested(argv), started=started)
+    except Exception as exc:
+        return print_error(EvalctlError("E_RUNNER_FAILED", f"internal error: {exc}", "run with --json and inspect errors[0]", 3), json_mode=json_mode or report_format_json_requested(argv), started=started)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())