agent-loop-guard-runtime 0.6.0a2__py3-none-any.whl

app/benchmark/dataset.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from app.benchmark.models import BenchmarkTask
+
+VALID_DIFFICULTIES = {"easy", "medium", "hard"}
+VALID_SCORERS = {"exact", "contains", "json_equal"}
+
+
+def bundled_dataset_path() -> Path:
+    return Path(__file__).with_name("data") / "starter-v1.json"
+
+
+def load_dataset(path: str | Path | None = None) -> tuple[dict[str, Any], list[BenchmarkTask]]:
+    source = Path(path) if path else bundled_dataset_path()
+    payload = json.loads(source.read_text(encoding="utf-8"))
+    errors = validate_dataset(payload)
+    if errors:
+        raise ValueError("; ".join(errors))
+    tasks = [BenchmarkTask(**item) for item in payload["tasks"]]
+    return payload, tasks
+
+
+def validate_dataset(payload: Any) -> list[str]:
+    if not isinstance(payload, dict):
+        return ["dataset must be a JSON object"]
+    errors: list[str] = []
+    if not isinstance(payload.get("version"), str) or not payload["version"].strip():
+        errors.append("version must be a non-empty string")
+    tasks = payload.get("tasks")
+    if not isinstance(tasks, list) or not tasks:
+        return errors + ["tasks must be a non-empty array"]
+    seen: set[str] = set()
+    for index, task in enumerate(tasks):
+        prefix = f"tasks[{index}]"
+        if not isinstance(task, dict):
+            errors.append(f"{prefix} must be an object")
+            continue
+        task_id = task.get("id")
+        if not isinstance(task_id, str) or not task_id:
+            errors.append(f"{prefix}.id must be a non-empty string")
+        elif task_id in seen:
+            errors.append(f"duplicate task id: {task_id}")
+        else:
+            seen.add(task_id)
+        if task.get("difficulty") not in VALID_DIFFICULTIES:
+            errors.append(f"{prefix}.difficulty must be easy, medium, or hard")
+        if not isinstance(task.get("prompt"), str) or not task["prompt"]:
+            errors.append(f"{prefix}.prompt must be a non-empty string")
+        scorer = str(task.get("scorer") or "exact")
+        if scorer not in VALID_SCORERS and not scorer.startswith("python:"):
+            errors.append(f"{prefix}.scorer is unsupported: {scorer}")
+        if "expected" not in task:
+            errors.append(f"{prefix}.expected is required")
+    return errors
+

app/benchmark/models.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+
+@dataclass(slots=True)
+class BenchmarkTask:
+    id: str
+    difficulty: str
+    prompt: str
+    expected: Any
+    scorer: str = "exact"
+    tags: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(slots=True)
+class AdapterResult:
+    output: str
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cost_micros: int = 0
+    error: str | None = None
+
+
+@dataclass(slots=True)
+class Observation:
+    run_id: str
+    candidate: str
+    task_id: str
+    difficulty: str
+    repetition: int
+    seed: int
+    score: float
+    duration_ms: int
+    input_tokens: int
+    output_tokens: int
+    cost_micros: int
+    output_hash: str
+    error: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+

app/benchmark/runner.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import hashlib
+import time
+import uuid
+from dataclasses import dataclass
+
+from app.benchmark.adapters import BenchmarkAdapter
+from app.benchmark.models import BenchmarkTask, Observation
+from app.benchmark.scorers import score_output
+
+
+@dataclass(slots=True)
+class RunLimits:
+    repetitions: int = 1
+    seed: int = 0
+    timeout_seconds: float = 30.0
+    token_budget: int = 0
+    cost_budget_micros: int = 0
+
+
+def run_benchmark(
+    tasks: list[BenchmarkTask],
+    adapter: BenchmarkAdapter,
+    candidate: str,
+    limits: RunLimits,
+) -> list[Observation]:
+    run_id = f"bench_{uuid.uuid4().hex[:24]}"
+    observations: list[Observation] = []
+    used_tokens = 0
+    used_cost = 0
+    for repetition in range(limits.repetitions):
+        for task in tasks:
+            if limits.token_budget and used_tokens >= limits.token_budget:
+                return observations
+            if limits.cost_budget_micros and used_cost >= limits.cost_budget_micros:
+                return observations
+            seed = limits.seed + repetition
+            started = time.perf_counter_ns()
+            result = adapter.execute(task, seed, limits.timeout_seconds)
+            duration_ms = max(0, int((time.perf_counter_ns() - started) / 1_000_000))
+            score = 0.0 if result.error else score_output(task.scorer, result.output, task.expected)
+            used_tokens += result.input_tokens + result.output_tokens
+            used_cost += result.cost_micros
+            observations.append(
+                Observation(
+                    run_id=run_id,
+                    candidate=candidate,
+                    task_id=task.id,
+                    difficulty=task.difficulty,
+                    repetition=repetition,
+                    seed=seed,
+                    score=score,
+                    duration_ms=duration_ms,
+                    input_tokens=result.input_tokens,
+                    output_tokens=result.output_tokens,
+                    cost_micros=result.cost_micros,
+                    output_hash=hashlib.sha256(result.output.encode("utf-8")).hexdigest(),
+                    error=result.error,
+                    metadata={"dataset.tags": task.tags},
+                )
+            )
+    return observations

app/benchmark/scorers.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import importlib
+import json
+from collections.abc import Callable
+from typing import Any
+
+
+def score_output(scorer: str, output: str, expected: Any) -> float:
+    if scorer == "exact":
+        return float(output.strip() == str(expected).strip())
+    if scorer == "contains":
+        return float(str(expected).strip().lower() in output.strip().lower())
+    if scorer == "json_equal":
+        try:
+            return float(json.loads(output) == expected)
+        except (json.JSONDecodeError, TypeError):
+            return 0.0
+    if scorer.startswith("python:"):
+        function = _load_custom_scorer(scorer.removeprefix("python:"))
+        value = float(function(output, expected))
+        return min(1.0, max(0.0, value))
+    raise ValueError(f"Unsupported scorer: {scorer}")
+
+
+def _load_custom_scorer(reference: str) -> Callable[[str, Any], float]:
+    module_name, separator, function_name = reference.partition(":")
+    if not separator:
+        raise ValueError("Custom scorer must use python:module:function")
+    function = getattr(importlib.import_module(module_name), function_name, None)
+    if not callable(function):
+        raise ValueError(f"Custom scorer is not callable: {reference}")
+    return function
+

app/benchmark/statistics.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import random
+from collections import defaultdict
+from statistics import mean
+from typing import Any
+
+
+def paired_bootstrap(
+    baseline: list[dict[str, Any]],
+    candidate: list[dict[str, Any]],
+    *,
+    samples: int = 2000,
+    seed: int = 0,
+    min_pairs: int = 10,
+    regression_threshold: float = 0.0,
+) -> dict[str, Any]:
+    left = _scores_by_key(baseline)
+    right = _scores_by_key(candidate)
+    keys = sorted(set(left) & set(right))
+    differences = [right[key] - left[key] for key in keys]
+    result: dict[str, Any] = {
+        "schema_version": "benchmark.compare.v1",
+        "pairs": len(differences),
+        "baseline_mean": mean(left[key] for key in keys) if keys else None,
+        "candidate_mean": mean(right[key] for key in keys) if keys else None,
+        "delta": mean(differences) if differences else None,
+        "confidence": 0.95,
+        "threshold": regression_threshold,
+    }
+    if len(differences) < min_pairs:
+        return {**result, "ci_low": None, "ci_high": None, "verdict": "inconclusive"}
+    rng = random.Random(seed)
+    bootstrapped = sorted(
+        mean(differences[rng.randrange(len(differences))] for _ in differences)
+        for _ in range(samples)
+    )
+    low = bootstrapped[int(samples * 0.025)]
+    high = bootstrapped[min(samples - 1, int(samples * 0.975))]
+    verdict = "regression" if high < -regression_threshold else "no_regression"
+    return {**result, "ci_low": low, "ci_high": high, "verdict": verdict}
+
+
+def _scores_by_key(rows: list[dict[str, Any]]) -> dict[tuple[str, int], float]:
+    grouped: dict[tuple[str, int], list[float]] = defaultdict(list)
+    for row in rows:
+        grouped[(str(row["task_id"]), int(row.get("repetition", 0)))].append(float(row["score"]))
+    return {key: mean(values) for key, values in grouped.items()}

app/benchmark/storage.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from app.benchmark.models import Observation
+
+
+def save_observations(rows: list[Observation], path: str | Path) -> Path:
+    destination = Path(path)
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    payload = [row.to_dict() for row in rows]
+    if destination.suffix.lower() == ".parquet":
+        try:
+            import pyarrow as pa
+            import pyarrow.parquet as pq
+        except ImportError as exc:
+            raise RuntimeError(
+                "Parquet output requires: pip install 'agent-loop-guard-runtime[bench]'"
+            ) from exc
+        pq.write_table(pa.Table.from_pylist(payload), destination)
+    else:
+        destination.write_text(
+            "".join(json.dumps(row, sort_keys=True) + "\n" for row in payload),
+            encoding="utf-8",
+        )
+    return destination
+
+
+def load_observations(path: str | Path) -> list[dict[str, Any]]:
+    source = Path(path)
+    if source.suffix.lower() == ".parquet":
+        try:
+            import pyarrow.parquet as pq
+        except ImportError as exc:
+            raise RuntimeError(
+                "Parquet input requires: pip install 'agent-loop-guard-runtime[bench]'"
+            ) from exc
+        return pq.read_table(source).to_pylist()
+    return [json.loads(line) for line in source.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def summarize_with_duckdb(path: str | Path) -> list[dict[str, Any]]:
+    try:
+        import duckdb
+    except ImportError as exc:
+        raise RuntimeError(
+            "DuckDB analysis requires: pip install 'agent-loop-guard-runtime[bench]'"
+        ) from exc
+    source = str(Path(path).resolve()).replace("'", "''")
+    reader = "read_parquet" if str(path).lower().endswith(".parquet") else "read_json_auto"
+    connection = duckdb.connect(":memory:")
+    rows = connection.execute(
+        f"SELECT difficulty, avg(score) AS score, count(*) AS observations "
+        f"FROM {reader}('{source}') GROUP BY difficulty ORDER BY difficulty"
+    ).fetchall()
+    return [
+        {"difficulty": difficulty, "score": score, "observations": observations}
+        for difficulty, score, observations in rows
+    ]
+
+
+def log_to_mlflow(rows: list[Observation], experiment: str = "agent-loop-guard") -> str:
+    try:
+        import mlflow
+    except ImportError as exc:
+        raise RuntimeError(
+            "MLflow tracking requires: pip install 'agent-loop-guard-runtime[bench]'"
+        ) from exc
+    scores = [row.score for row in rows]
+    mlflow.set_experiment(experiment)
+    with mlflow.start_run() as run:
+        mlflow.log_param("candidate", rows[0].candidate if rows else "unknown")
+        mlflow.log_param("observations", len(rows))
+        mlflow.log_metric("score", sum(scores) / len(scores) if scores else 0.0)
+        mlflow.log_metric("tokens", sum(row.input_tokens + row.output_tokens for row in rows))
+        return run.info.run_id