evalwarden 0.6.0__py3-none-any.whl

evalwarden/__init__.py

@@ -0,0 +1,3 @@
+"""evalwarden: a linter for agent evaluations."""
+
+__version__ = "0.6.0"

evalwarden/adapters/__init__.py

@@ -0,0 +1,70 @@
+"""Adapter protocol and registry.
+
+Adapters are the ONLY harness-specific code in the project. They discover and
+translate; the core never becomes a harness. Each adapter:
+
+- is read-only (never mutates the input files),
+- works offline (no network in default mode),
+- preserves source locations,
+- pins its own version and the schema versions it understands,
+- reports unsupported fields explicitly instead of silently dropping them.
+
+v0.1 ships one adapter: Inspect AI style eval artifacts. Promptfoo, Harbor,
+and BrowserGym adapters plug into REGISTRY later with the same contract.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Protocol
+
+from ..model import Confidence, IntegrityModel
+
+
+class AuditError(Exception):
+    """The audit could not complete (exit code 2)."""
+
+
+class Adapter(Protocol):
+    name: str
+    version: str
+
+    def detect(self, path: Path) -> Confidence:
+        """How confident are we that this adapter understands `path`?"""
+        ...
+
+    def collect(self, path: Path) -> dict:
+        """Read-only collection of raw evidence from the artifact."""
+        ...
+
+    def normalize(self, bundle: dict) -> IntegrityModel:
+        """Translate raw evidence into the framework-neutral integrity model."""
+        ...
+
+
+REGISTRY: list[Adapter] = []
+
+
+def register(adapter: Adapter) -> Adapter:
+    # Accept either an instance or a class (instantiated here) so adapters can
+    # use either `@register` on the class or `register(MyAdapter())`.
+    REGISTRY.append(adapter() if isinstance(adapter, type) else adapter)
+    return adapter
+
+
+def autodetect(path: Path) -> Adapter:
+    """Pick the most confident adapter for `path`, or raise AuditError."""
+    if not REGISTRY:
+        raise AuditError("no adapters registered")
+    ranked = sorted(
+        ((adapter.detect(path), adapter) for adapter in REGISTRY),
+        key=lambda item: (item[0] == Confidence.HIGH, item[0] == Confidence.MEDIUM),
+        reverse=True,
+    )
+    confidence, adapter = ranked[0]
+    if confidence == Confidence.LOW:
+        raise AuditError(
+            f"no adapter recognizes {path} "
+            f"(best guess: {adapter.name}, confidence=low). "
+            "Expected an eval artifact directory (see demo/leaky for the layout)."
+        )
+    return adapter

evalwarden/adapters/inspect_ai.py

@@ -0,0 +1,213 @@
+"""Inspect AI adapter (v0.1).
+
+Reads an Inspect-style eval artifact directory -- the v0.1 normalized input
+format, modeled on Inspect's Task / dataset / scorer / .eval-log concepts:
+
+    eval-artifact/
+      dataset.json       samples: [{id, prompt, metadata}]
+      environment.json   env vars visible to the solver, mounts
+      grader.json        verifier config, pass conditions, tests
+      run.json           per-task attempts with status, usage, actions
+      judge_run.json     (optional) model-judge config, judgments, reference labels
+
+This is a read-only translation layer. Full-fidelity parsing of real Inspect
+`.eval` logs is a later milestone; the adapter pins the schema version it
+understands and fails clearly on anything else.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+from ..model import (
+    Attempt,
+    Confidence,
+    Environment,
+    Grader,
+    IntegrityModel,
+    Judgment,
+    Mount,
+    TaskSample,
+)
+from . import AuditError, register
+
+ADAPTER_NAME = "inspect"
+ADAPTER_VERSION = "0.1.0"
+SCHEMA_VERSION = "evalwarden-artifact-v1"
+
+
+def _read_json(path: Path) -> Any:
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise AuditError(f"missing required file: {path}") from exc
+    except json.JSONDecodeError as exc:
+        raise AuditError(f"invalid JSON in {path}: {exc}") from exc
+
+
+def _digest(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+@register
+class InspectAdapter:
+    name = ADAPTER_NAME
+    version = ADAPTER_VERSION
+
+    def detect(self, path: Path) -> Confidence:
+        if not path.is_dir():
+            return Confidence.LOW
+        has_dataset = (path / "dataset.json").is_file()
+        has_grader = (path / "grader.json").is_file()
+        if has_dataset and has_grader:
+            return Confidence.HIGH
+        if has_dataset:
+            return Confidence.MEDIUM
+        return Confidence.LOW
+
+    def collect(self, path: Path) -> dict:
+        """Read-only: files are opened for reading and never modified."""
+        bundle: dict[str, Any] = {"root": path}
+        digests: dict[str, str] = {}
+        for fname in (
+            "dataset.json",
+            "environment.json",
+            "grader.json",
+            "run.json",
+            "judge_run.json",
+        ):
+            fpath = path / fname
+            if fpath.is_file():
+                bundle[fname] = _read_json(fpath)
+                digests[fname] = _digest(fpath)
+        for fname in ("gold_map.json",):
+            fpath = path / fname
+            if fpath.is_file():
+                # Recorded for the digest manifest only; never parsed for content
+                # beyond its existence (it is the leak, not the evidence).
+                digests[fname] = _digest(fpath)
+                bundle.setdefault("extra_files", []).append(fname)
+        bundle["digests"] = digests
+        return bundle
+
+    def normalize(self, bundle: dict) -> IntegrityModel:
+        root: Path = bundle["root"]
+        dataset = bundle.get("dataset.json", {})
+        environment = bundle.get("environment.json", {})
+        grader_cfg = bundle.get("grader.json", {})
+        run = bundle.get("run.json", {})
+
+        if dataset.get("schema_version", SCHEMA_VERSION) != SCHEMA_VERSION and "tasks" not in dataset:
+            raise AuditError(
+                f"unsupported dataset schema (expected {SCHEMA_VERSION!r} with a 'tasks' list)"
+            )
+
+        tasks = [
+            TaskSample(
+                id=str(t.get("id", f"task-{i}")),
+                prompt=str(t.get("prompt", "")),
+                metadata=dict(t.get("metadata", {})),
+            )
+            for i, t in enumerate(dataset.get("tasks", []))
+        ]
+
+        env_vars = {
+            str(name): "<redacted>"  # values are never stored; names are the signal
+            for name in (environment.get("env") or {})
+        }
+        mounts = [
+            Mount(
+                path=str(m.get("path", "")),
+                mode=str(m.get("mode", "ro")),
+                agent_access=str(m.get("agent_access", "read")),
+            )
+            for m in (environment.get("mounts") or [])
+        ]
+
+        verifier = grader_cfg.get("verifier") or {}
+        judge = grader_cfg.get("judge") or {}
+        grader = Grader(
+            kind=str(grader_cfg.get("kind", "script")),
+            verifier_path=verifier.get("path"),
+            verifier_writable_by_agent=bool(verifier.get("writable_by_agent", False)),
+            accepts_empty_output=bool(grader_cfg.get("accepts_empty_output", False)),
+            tests=[str(t) for t in (grader_cfg.get("tests") or [])],
+            judge_model=judge.get("model"),
+            judge_family=judge.get("family"),
+            protocol=judge.get("protocol"),
+            counterbalanced=judge.get("counterbalanced"),
+            temperature=judge.get("temperature"),
+            repeats=int(judge.get("repeats", 1)),
+            rubric_criteria=[str(c) for c in (judge.get("rubric_criteria") or [])],
+            scale_anchors={str(k): str(v) for k, v in (judge.get("scale_anchors") or {}).items()},
+        )
+
+        attempts = [
+            Attempt(
+                task_id=str(a.get("task_id", "")),
+                status=str(a.get("status", "error")),
+                score=a.get("score"),
+                tool_calls=int(a.get("tool_calls", 0)),
+                actions=[str(x) for x in (a.get("actions") or [])],
+                tokens_in=a.get("tokens_in"),
+                tokens_out=a.get("tokens_out"),
+                latency_s=a.get("latency_s"),
+                tries=int(a.get("tries", 1)),
+                empty_submission=bool(a.get("empty_submission", False)),
+            )
+            for a in (run.get("attempts") or [])
+        ]
+
+        unsupported: list[str] = []
+
+        judge_run = bundle.get("judge_run.json", {})
+        judgments: list[Judgment] = []
+        for j in judge_run.get("judgments", []):
+            candidates = sorted(str(c) for c in (j.get("candidates") or []))
+            judgments.append(
+                Judgment(
+                    task_id=str(j.get("task_id", "")),
+                    candidates=candidates,
+                    presentation_order=[str(c) for c in (j.get("presentation_order") or candidates)],
+                    winner=j.get("winner"),
+                    scores={str(k): float(v) for k, v in (j.get("scores") or {}).items()},
+                    lengths={str(k): int(v) for k, v in (j.get("lengths") or {}).items()},
+                    repeat_index=int(j.get("repeat_index", 0)),
+                )
+            )
+        if any("rationale" in j for j in judge_run.get("judgments", [])):
+            # Raw judge text is redacted at the boundary by design, not parsed.
+            # Reported here so the redaction is explicit, not silent.
+            unsupported.append("judge_run.json:judgments[].rationale (redacted: raw judge text not stored)")
+        grader.reference_labels = {
+            str(k): str(v) for k, v in (judge_run.get("reference_labels") or {}).items()
+        }
+        for fname, content in bundle.items():
+            if fname in ("root", "digests", "extra_files"):
+                continue
+            if isinstance(content, dict):
+                known = {
+                    "dataset.json": {"schema_version", "eval_id", "tasks"},
+                    "environment.json": {"env", "mounts", "notes"},
+                    "grader.json": {"kind", "verifier", "accepts_empty_output", "tests", "judge"},
+                    "run.json": {"solver", "attempts", "notes"},
+                    "judge_run.json": {"judge", "judgments", "reference_labels", "notes"},
+                }.get(fname, set())
+                for key in content:
+                    if key not in known:
+                        unsupported.append(f"{fname}:{key}")
+
+        return IntegrityModel(
+            eval_id=str(dataset.get("eval_id", root.name)),
+            adapter_name=self.name,
+            adapter_version=self.version,
+            tasks=tasks,
+            environment=Environment(env_vars=env_vars, mounts=mounts),
+            grader=grader,
+            attempts=attempts,
+            judgments=judgments,
+            unsupported=unsupported,
+            digests=dict(bundle.get("digests", {})),
+        )

evalwarden/adapters/promptfoo.py

@@ -0,0 +1,357 @@
+"""Promptfoo adapter (v0.5).
+
+Reads a Promptfoo eval artifact directory:
+
+    eval-artifact/
+      promptfooconfig.yaml   (or .yml) -- prompts, providers, tests, assertions, env
+      results.json           (optional) -- JSON export from `promptfoo eval -o results.json`
+
+Minimum viable input per the spec: promptfooconfig.yaml plus the JSON export.
+A config-only audit is valid but incomplete (no runs to analyze). When only
+results.json is present, the config embedded in its envelope is used and the
+substitution is reported explicitly.
+
+Field names follow promptfoo's documented output schema (OutputFile envelope
+with results.version == 3; per-result success/score/error, vars,
+response.output, response.tokenUsage.{prompt,completion,total}, latencyMs,
+gradingResult.{pass,score,reason,componentResults}). Anything else lands in
+`unsupported`, never silently dropped.
+
+This is a read-only translation layer: it parses the config and the recorded
+results. It never executes an eval.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+from ..model import (
+    Attempt,
+    Confidence,
+    Environment,
+    Grader,
+    IntegrityModel,
+    TaskSample,
+)
+from . import AuditError, register
+
+ADAPTER_NAME = "promptfoo"
+ADAPTER_VERSION = "0.6.0"
+# Promptfoo's documented results envelope version (OutputFile.results.version).
+RESULTS_VERSION = 3
+
+CONFIG_NAMES = ("promptfooconfig.yaml", "promptfooconfig.yml")
+RESULTS_NAME = "results.json"
+
+# Top-level config keys the adapter understands. Everything else is reported
+# in `unsupported` so coverage gaps are explicit, not silent.
+KNOWN_CONFIG_KEYS = {
+    "description",
+    "env",
+    "prompts",
+    "providers",
+    "defaultTest",
+    "scenarios",
+    "tests",
+    "metadata",
+    "outputPath",
+    "writeLatestResults",
+    "sharing",
+}
+
+
+def _envelope_has_config(results_path: Path) -> bool:
+    """Cheap peek for detect(): does the results envelope carry a config dict?"""
+    try:
+        envelope = json.loads(results_path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return False
+    return isinstance(envelope, dict) and isinstance(envelope.get("config"), dict)
+
+
+def _read_json(path: Path) -> Any:
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise AuditError(f"missing required file: {path}") from exc
+    except json.JSONDecodeError as exc:
+        raise AuditError(f"invalid JSON in {path}: {exc}") from exc
+
+
+def _read_yaml(path: Path) -> Any:
+    try:
+        import yaml
+    except ImportError as exc:
+        raise AuditError(
+            "promptfoo adapter needs PyYAML to parse promptfooconfig.yaml "
+            "(pip install 'evalwarden' again, or pip install pyyaml)"
+        ) from exc
+    try:
+        return yaml.safe_load(path.read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise AuditError(f"missing required file: {path}") from exc
+    except yaml.YAMLError as exc:
+        raise AuditError(f"invalid YAML in {path}: {exc}") from exc
+
+
+def _digest(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _config_path(path: Path) -> Path | None:
+    for name in CONFIG_NAMES:
+        candidate = path / name
+        if candidate.is_file():
+            return candidate
+    return None
+
+
+@register
+class PromptfooAdapter:
+    name = ADAPTER_NAME
+    version = ADAPTER_VERSION
+
+    def detect(self, path: Path) -> Confidence:
+        if not path.is_dir():
+            return Confidence.LOW
+        has_config = _config_path(path) is not None
+        results_path = path / RESULTS_NAME
+        has_results = results_path.is_file()
+        if has_config and has_results:
+            return Confidence.HIGH
+        if has_config:
+            # Config-only audit: valid but incomplete (no runs to analyze).
+            return Confidence.MEDIUM
+        if has_results and _envelope_has_config(results_path):
+            # No promptfooconfig.yaml, but the results envelope carries the
+            # config it was produced from.
+            return Confidence.MEDIUM
+        return Confidence.LOW
+
+    def collect(self, path: Path) -> dict:
+        """Read-only: files are opened for reading and never modified."""
+        bundle: dict[str, Any] = {"root": path}
+        digests: dict[str, str] = {}
+        config_path = _config_path(path)
+        if config_path is not None:
+            bundle["config"] = _read_yaml(config_path)
+            bundle["config_file"] = config_path.name
+            digests[config_path.name] = _digest(config_path)
+        results_path = path / RESULTS_NAME
+        if results_path.is_file():
+            bundle["results"] = _read_json(results_path)
+            digests[RESULTS_NAME] = _digest(results_path)
+        bundle["digests"] = digests
+        return bundle
+
+    def normalize(self, bundle: dict) -> IntegrityModel:
+        root: Path = bundle["root"]
+        config = bundle.get("config") or {}
+        config_file = bundle.get("config_file", "promptfooconfig.yaml")
+        envelope = bundle.get("results") or {}
+        inner = envelope.get("results") or {}
+
+        if not isinstance(config, dict):
+            raise AuditError(f"{config_file}: top-level mapping expected")
+
+        unsupported: list[str] = []
+        if "config" not in bundle and envelope.get("config") is not None:
+            # Graceful degradation: no promptfooconfig.yaml, but the results
+            # envelope carries the eval config it was produced from.
+            config = envelope["config"]
+            unsupported.append(
+                f"{RESULTS_NAME}: promptfooconfig.yaml absent; "
+                "using the config embedded in the results envelope"
+            )
+
+        version = inner.get("version")
+        if "results" in bundle and version != RESULTS_VERSION:
+            raise AuditError(
+                f"{RESULTS_NAME}: unsupported results version {version!r} "
+                f"(this adapter understands version {RESULTS_VERSION})"
+            )
+
+        eval_id = (
+            config.get("description")
+            or envelope.get("evalId")
+            or root.name
+        )
+
+        tasks = _normalize_tests(config, unsupported)
+        env_vars = {
+            str(name): "<redacted>"  # values are never stored; names are the signal
+            for name in (config.get("env") or {})
+        }
+        grader = _normalize_grader(config)
+        attempts = _normalize_attempts(inner)
+
+        for key in config:
+            if key not in KNOWN_CONFIG_KEYS:
+                unsupported.append(f"{config_file}:{key}")
+        if config.get("scenarios"):
+            unsupported.append(
+                f"{config_file}:scenarios (scenario-generated tests not expanded)"
+            )
+        for test in _inline_tests(config):
+            for key in test:
+                if key not in {
+                    "description", "vars", "assert", "options",
+                    "threshold", "metadata",
+                }:
+                    unsupported.append(f"{config_file}:tests[]:{key}")
+
+        return IntegrityModel(
+            eval_id=str(eval_id),
+            adapter_name=self.name,
+            adapter_version=self.version,
+            tasks=tasks,
+            environment=Environment(env_vars=env_vars, mounts=[]),
+            grader=grader,
+            attempts=attempts,
+            unsupported=unsupported,
+            digests=dict(bundle.get("digests", {})),
+            # Checks cite canonical artifact names; point them at the real files.
+            file_aliases={
+                "environment.json": config_file,
+                "grader.json": config_file,
+                "judge_run.json": config_file,
+                "run.json": RESULTS_NAME,
+            },
+        )
+
+
+def _inline_tests(config: dict) -> list[dict]:
+    """Tests declared inline in the config. file:// and other references are
+    left for `unsupported`: expanding them would pull arbitrary files into the
+    audit, and the declared config is the contract under review."""
+    tests = config.get("tests") or []
+    if isinstance(tests, dict):
+        tests = [tests]
+    return [t for t in tests if isinstance(t, dict)]
+
+
+def _normalize_tests(config: dict, unsupported: list[str]) -> list[TaskSample]:
+    tasks: list[TaskSample] = []
+    raw_tests = config.get("tests")
+    if isinstance(raw_tests, str):
+        unsupported.append(f"tests: {raw_tests} (external test file not expanded)")
+        return tasks
+    if isinstance(raw_tests, list) and any(
+        isinstance(t, str) for t in raw_tests
+    ):
+        unsupported.append("tests[]: file:// references not expanded")
+    for i, test in enumerate(_inline_tests(config)):
+        vars_ = test.get("vars") or {}
+        assertions = [str(a.get("type", "?")) for a in (test.get("assert") or []) if isinstance(a, dict)]
+        tasks.append(
+            TaskSample(
+                id=str(test.get("description") or f"test-{i}"),
+                prompt=str(test.get("description") or ""),
+                metadata={
+                    "vars": sorted(str(k) for k in vars_),
+                    "assertions": assertions,
+                },
+            )
+        )
+    return tasks
+
+
+def _assertions(config: dict) -> list[dict]:
+    """All assertions: shared defaultTest ones plus per-test ones."""
+    found: list[dict] = []
+    default = config.get("defaultTest") or {}
+    if isinstance(default, dict):
+        found.extend(a for a in (default.get("assert") or []) if isinstance(a, dict))
+    for test in _inline_tests(config):
+        found.extend(a for a in (test.get("assert") or []) if isinstance(a, dict))
+    return found
+
+
+def _provider_id(ref: Any) -> str | None:
+    if isinstance(ref, str):
+        return ref
+    if isinstance(ref, dict) and ref.get("id"):
+        return str(ref["id"])
+    return None
+
+
+def _grader_provider(config: dict) -> tuple[str | None, float | None]:
+    """The model grading llm-rubric assertions: per-test options.provider wins,
+    then defaultTest.options.provider. Returns (provider_id, temperature)."""
+    for test in _inline_tests(config):
+        options = test.get("options") or {}
+        if isinstance(options, dict) and options.get("provider") is not None:
+            return _provider_id(options["provider"]), _provider_temperature(options["provider"])
+    default = config.get("defaultTest") or {}
+    options = default.get("options") or {} if isinstance(default, dict) else {}
+    if isinstance(options, dict) and options.get("provider") is not None:
+        return _provider_id(options["provider"]), _provider_temperature(options["provider"])
+    return None, None
+
+
+def _provider_temperature(ref: Any) -> float | None:
+    if isinstance(ref, dict):
+        cfg = ref.get("config") or {}
+        if isinstance(cfg, dict) and isinstance(cfg.get("temperature"), (int, float)):
+            return float(cfg["temperature"])
+    return None
+
+
+def _normalize_grader(config: dict) -> Grader:
+    assertions = _assertions(config)
+    types = sorted({str(a.get("type", "?")) for a in assertions})
+    is_judge = "llm-rubric" in types
+    grader = Grader(
+        kind="judge" if is_judge else "script",
+        tests=types,
+    )
+    if is_judge:
+        provider_id, temperature = _grader_provider(config)
+        grader.judge_model = provider_id
+        grader.temperature = temperature
+        # Each llm-rubric value is the rubric; promptfoo has no anchored scale
+        # levels, which JUDGE-001 reports as an unanchored rubric.
+        grader.rubric_criteria = [
+            str(a["value"]) for a in assertions
+            if str(a.get("type")) == "llm-rubric" and a.get("value")
+        ]
+    return grader
+
+
+def _normalize_attempts(inner: dict) -> list[Attempt]:
+    attempts: list[Attempt] = []
+    rows = inner.get("results") or []
+    for i, row in enumerate(rows):
+        if not isinstance(row, dict):
+            continue
+        error = row.get("error")
+        success = row.get("success")
+        status = "error" if error else ("pass" if success else "fail")
+        response = row.get("response") or {}
+        usage = response.get("tokenUsage") or {}
+        latency_ms = row.get("latencyMs")
+        output = response.get("output")
+        attempts.append(
+            Attempt(
+                task_id=str(row.get("description") or f"test-{row.get('testIdx', i)}"),
+                status=status,
+                score=_to_float(row.get("score")),
+                tokens_in=_to_int(usage.get("prompt")),
+                tokens_out=_to_int(usage.get("completion")),
+                latency_s=(float(latency_ms) / 1000.0) if isinstance(latency_ms, (int, float)) else None,
+                # An explicitly empty model output scored as a pass is the
+                # GRAD-002 signal; anything else is just a short answer.
+                empty_submission=isinstance(output, str) and output == "" and status == "pass",
+            )
+        )
+    return attempts
+
+
+def _to_float(value: Any) -> float | None:
+    return float(value) if isinstance(value, (int, float)) else None
+
+
+def _to_int(value: Any) -> int | None:
+    return int(value) if isinstance(value, (int, float)) else None

evalwarden/checks/__init__.py

@@ -0,0 +1,27 @@
+"""Rule registry. v0.3 ships deterministic, high-precision checks only.
+
+Sequencing rule: earn trust with deterministic evidence before adding
+probabilistic signals. Every finding carries a confidence label; a linter that
+cries contamination on a clean eval is worse than no auditor.
+"""
+from __future__ import annotations
+
+from .base import Check
+from .cost import CHECKS as COST_CHECKS
+from .env_leakage import EnvLeakageCheck
+from .grader import EmptyPathCheck, VerifierWritableCheck
+from .judge import CHECKS as JUDGE_CHECKS
+
+REGISTRY: list[Check] = [
+    EnvLeakageCheck(),  # ENV-001
+    VerifierWritableCheck(),  # GRAD-001
+    EmptyPathCheck(),  # GRAD-002
+    *COST_CHECKS,  # COST-001 .. COST-004
+    *JUDGE_CHECKS,  # JUDGE-001 .. JUDGE-006
+]
+
+BY_ID: dict[str, Check] = {check.meta.id: check for check in REGISTRY}
+
+
+def get_check(check_id: str) -> Check | None:
+    return BY_ID.get(check_id.upper())