gatewaysdk 0.3.2__py3-none-any.whl

gatewaysdk/benchmark_hub/path_utils.py

@@ -0,0 +1,81 @@
+"""Eval-results dir layout + resume matching.
+
+Ported (stdlib-only) from Prime Intellect's ``verifiers`` (MIT). A run's results
+live at ``<output_dir>/evals/<env_id>--<model>/<run_id>/`` holding
+``results.jsonl`` + ``metadata.json``. Resume finds the newest *incomplete* run
+whose saved config matches, counting completed rollouts per example (clamped to
+``rollouts_per_example``) — so a crashed run continues instead of restarting.
+"""
+
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+from typing import Any, Dict, Optional, Union
+
+from .save_utils import (
+    METADATA_FILE,
+    RESULTS_FILE,
+    load_metadata,
+    load_outputs,
+)
+
+
+def _model_slug(model: str) -> str:
+    return model.replace("/", "--")
+
+
+def get_results_path(output_dir: Union[str, Path], env_id: str, model: str) -> Path:
+    """A fresh results dir for a run: ``<output_dir>/evals/<env>--<model>/<uuid8>``."""
+    run_id = uuid.uuid4().hex[:8]
+    return Path(output_dir) / "evals" / f"{env_id}--{_model_slug(model)}" / run_id
+
+
+def is_valid_eval_results_path(path: Union[str, Path]) -> bool:
+    """A results dir is valid iff it has BOTH results.jsonl and metadata.json."""
+    p = Path(path)
+    return (p / RESULTS_FILE).exists() and (p / METADATA_FILE).exists()
+
+
+def count_saved_rollouts(results_path: Union[str, Path], rollouts_per_example: int) -> Dict[Any, int]:
+    """Completed rollouts per ``example_id``, clamped to ``rollouts_per_example``."""
+    counts: Dict[Any, int] = {}
+    rpe = rollouts_per_example if rollouts_per_example > 0 else 1
+    for row in load_outputs(results_path):
+        eid = row.get("example_id")
+        if eid is None:
+            continue
+        counts[eid] = min(counts.get(eid, 0) + 1, rpe)
+    return counts
+
+
+def find_latest_incomplete_eval_results_path(
+    output_dir: Union[str, Path],
+    env_id: str,
+    model: str,
+    num_examples: int,
+    rollouts_per_example: int,
+    shuffle: bool = False,
+    shuffle_seed: Optional[int] = None,
+) -> Optional[Path]:
+    """Newest results dir (by mtime) whose saved metadata matches this config AND
+    that still has fewer saved rollouts than requested — or None."""
+    root = Path(output_dir) / "evals" / f"{env_id}--{_model_slug(model)}"
+    if not root.exists():
+        return None
+    candidates = [d for d in root.iterdir() if d.is_dir() and is_valid_eval_results_path(d)]
+    candidates.sort(key=lambda d: d.stat().st_mtime, reverse=True)
+
+    total = num_examples * rollouts_per_example if num_examples > 0 else None
+    for c in candidates:
+        meta = load_metadata(c)
+        if meta.get("env_id") != env_id or meta.get("model") != model:
+            continue
+        if meta.get("rollouts_per_example") != rollouts_per_example:
+            continue
+        if meta.get("shuffle") != shuffle or meta.get("shuffle_seed") != (shuffle_seed):
+            continue
+        saved = sum(count_saved_rollouts(c, rollouts_per_example).values())
+        if total is None or saved < total:
+            return c
+    return None

gatewaysdk/benchmark_hub/save_utils.py

@@ -0,0 +1,101 @@
+"""Local eval-results persistence: ``results.jsonl`` + ``metadata.json``.
+
+Ported (stdlib-only) from Prime Intellect's ``verifiers`` (MIT) save layer so a
+``bench eval`` run writes the same on-disk contract that powers resume + upload.
+``save_metadata`` is atomic (temp file in the same dir + ``os.replace``) and drops
+the non-serializable ``path_to_save``/``date`` keys, matching verifiers.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+RESULTS_FILE = "results.jsonl"
+METADATA_FILE = "metadata.json"
+
+
+def make_serializable(obj: Any) -> Any:
+    """Best-effort JSON-serializable copy: recurse dict/list, stringify the rest."""
+    try:
+        json.dumps(obj)
+        return obj
+    except (TypeError, ValueError):
+        if isinstance(obj, dict):
+            return {k: make_serializable(v) for k, v in obj.items()}
+        if isinstance(obj, (list, tuple)):
+            return [make_serializable(v) for v in obj]
+        return str(obj)
+
+
+def save_outputs(
+    results_path: Union[str, Path],
+    rows: List[Dict[str, Any]],
+    mode: str = "a",
+) -> None:
+    """Append (or overwrite) rollout rows to ``results.jsonl``, one per line."""
+    f = Path(results_path) / RESULTS_FILE
+    f.parent.mkdir(parents=True, exist_ok=True)
+    with open(f, mode) as fh:
+        for row in rows:
+            fh.write(json.dumps(make_serializable(row)) + "\n")
+
+
+def load_outputs(results_path: Union[str, Path]) -> List[Dict[str, Any]]:
+    """Read ``results.jsonl``, skipping blank/malformed/non-dict lines (a torn last
+    line from a crash is silently ignored — resume-safe)."""
+    f = Path(results_path) / RESULTS_FILE
+    if not f.exists():
+        return []
+    out: List[Dict[str, Any]] = []
+    for line in f.read_text().splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            row = json.loads(line)
+        except Exception:
+            continue
+        if isinstance(row, dict):
+            out.append(row)
+    return out
+
+
+def save_metadata(results_path: Union[str, Path], metadata: Dict[str, Any]) -> None:
+    """Atomically write ``metadata.json`` (temp in same dir + ``os.replace``),
+    dropping ``path_to_save``/``date`` (non-serializable, per verifiers)."""
+    p = Path(results_path)
+    p.mkdir(parents=True, exist_ok=True)
+    meta = dict(metadata)
+    meta.pop("path_to_save", None)
+    meta.pop("date", None)
+    fd, tmp = tempfile.mkstemp(dir=str(p), suffix=".tmp")
+    try:
+        with os.fdopen(fd, "w") as fh:
+            json.dump(make_serializable(meta), fh, indent=2)
+        os.replace(tmp, str(p / METADATA_FILE))
+    finally:
+        if os.path.exists(tmp):
+            os.unlink(tmp)
+
+
+def load_metadata(results_path: Union[str, Path]) -> Dict[str, Any]:
+    """Read ``metadata.json`` ({} if absent/malformed)."""
+    f = Path(results_path) / METADATA_FILE
+    if not f.exists():
+        return {}
+    try:
+        data = json.loads(f.read_text())
+        return data if isinstance(data, dict) else {}
+    except Exception:
+        return {}
+
+
+def validate_resume_metadata(saved_metadata: Dict[str, Any], requested_num_examples: int) -> None:
+    """On resume, ``num_examples`` may only grow — a smaller request is an error."""
+    saved_n = saved_metadata.get("num_examples")
+    if isinstance(saved_n, int) and saved_n > 0 and requested_num_examples > 0 and requested_num_examples < saved_n:
+        raise ValueError(f"cannot resume: requested num_examples {requested_num_examples} " f"< saved {saved_n}")

gatewaysdk/benchmark_hub/verifiers_adapter.py

@@ -0,0 +1,327 @@
+"""Score a verifiers-format container and harvest into Gateway.
+
+`verifiers` (MIT) is the one genuinely open, self-hostable piece of Prime's eval
+stack: ``Environment.evaluate_sync()`` runs the rollout loop against ANY
+OpenAI-compatible client (ours) and scores via the rubric — so we don't reimplement
+rollouts/parsers/scoring. This adapter maps its ``GenerateOutputs`` into our
+version-pinned eval sample shape and pushes it through ``GatewayEvalsClient``.
+
+Verified against verifiers ``Environment.evaluate`` →
+``GenerateOutputs{outputs: list[RolloutOutput], metadata}``; each ``RolloutOutput``
+carries ``prompt / completion / answer / reward`` and the named sub-rewards in
+``metrics: dict[str, float]``. ``example_id`` is declared on ``RolloutOutput`` but
+may be absent on rows produced by ``evaluate_sync``; the mapper falls back to a
+positional example index derived from ``metadata.rollouts_per_example``.
+
+The verifiers import is lazy (optional dependency); the mappers are pure + tested.
+
+NOTE (verifiers v1): this adapter drives the LEGACY v0 API (`load_environment` +
+`Environment.evaluate_sync`), which the v1 release froze — it only keeps working
+with `verifiers<1.0.0` installed. v1 Taskset packages (`verifiers.v1` namespace,
+detected as kind "verifiers-v1" by `detect_env_kind`) are run by the platform
+benchmark runner's verifiers-v1 engine (deploy/benchmark-runner/engines/
+verifiers_v1.py); a v1 in-process path here is a deliberate non-goal until the
+v1 API settles.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, Dict, List, Optional
+
+from .evals import EvalHarvestError, GatewayEvalsClient
+
+
+def _get(obj: Any, key: str, default: Any = None) -> Any:
+    """Read a field from a dict (TypedDict) or an attribute object."""
+    if isinstance(obj, dict):
+        return obj.get(key, default)
+    return getattr(obj, key, default)
+
+
+def _as_text(value: Any) -> Optional[str]:
+    if value is None:
+        return None
+    if isinstance(value, str):
+        return value
+    try:
+        return json.dumps(value, default=str)
+    except Exception:
+        return str(value)
+
+
+def _rollouts(outputs: Any) -> List[Any]:
+    return _get(outputs, "outputs", []) or []
+
+
+def samples_from_generate_outputs(outputs: Any) -> List[Dict[str, Any]]:
+    """Map verifiers ``GenerateOutputs`` → our eval-sample dicts. The per-rollout
+    ``metrics`` (named rubric components) become the multi-reward ``rewards`` map.
+
+    ``rolloutIndex`` is the occurrence of a rollout *within its example*. verifiers'
+    ``RolloutOutput`` lists ``example_id`` as a field, but rows produced by
+    ``evaluate_sync`` are not guaranteed to carry it as a top-level key — so when it
+    is missing we fall back to the positional example index
+    (``i // rollouts_per_example``) instead of letting every id collapse to the string
+    ``"None"``, which would flatten all rollout indices into one monotonic run across
+    the whole batch."""
+    rollouts = _rollouts(outputs)
+    rpe = int(_get(_get(outputs, "metadata", {}) or {}, "rollouts_per_example", 1) or 1)
+    if rpe <= 0:
+        rpe = 1
+    seen: Dict[str, int] = {}
+    samples: List[Dict[str, Any]] = []
+    for i, r in enumerate(rollouts):
+        raw_example_id = _get(r, "example_id")
+        example_id = str(raw_example_id if raw_example_id is not None else i // rpe)
+        idx = seen.get(example_id, 0)
+        seen[example_id] = idx + 1
+
+        reward = _get(r, "reward")
+        answer = _get(r, "answer")
+        sample: Dict[str, Any] = {
+            "exampleId": example_id,
+            "rolloutIndex": idx,
+            "prompt": _as_text(_get(r, "prompt")),
+            "completion": _as_text(_get(r, "completion")),
+            "answer": answer if isinstance(answer, str) else _as_text(answer),
+            "reward": float(reward) if reward is not None else None,
+            "score": float(reward) if reward is not None else None,
+            "rewards": _get(r, "metrics") or {},  # named sub-rewards
+            "info": _get(r, "info") or {},
+        }
+        samples.append({k: v for k, v in sample.items() if v is not None})
+    return samples
+
+
+def metrics_from_generate_outputs(outputs: Any) -> Dict[str, Any]:
+    """Aggregate metrics: average reward + average of each named sub-reward."""
+    rollouts = _rollouts(outputs)
+    rewards = [float(_get(r, "reward")) for r in rollouts if _get(r, "reward") is not None]
+    by_metric: Dict[str, List[float]] = {}
+    for r in rollouts:
+        for name, value in (_get(r, "metrics") or {}).items():
+            if isinstance(value, (int, float)):
+                by_metric.setdefault(name, []).append(float(value))
+
+    def _mean(xs: List[float]) -> Optional[float]:
+        return sum(xs) / len(xs) if xs else None
+
+    return {
+        "n": len(rollouts),
+        "avg_reward": _mean(rewards),
+        "metrics": {name: _mean(vals) for name, vals in by_metric.items()},
+    }
+
+
+def write_results_artifact(outputs: Any, config: Any, results_path: Any) -> Dict[str, Any]:
+    """Write the verifiers-compatible on-disk artifact for a completed run —
+    ``results.jsonl`` (raw rollout rows, keyed on ``example_id`` for resume) +
+    ``metadata.json`` (config + aggregates). ``config`` is an ``EvalConfig``.
+    Returns the metadata dict."""
+    from .save_utils import save_metadata, save_outputs
+
+    save_outputs(results_path, _rollouts(outputs), mode="w")
+    agg = metrics_from_generate_outputs(outputs)
+    metadata: Dict[str, Any] = {
+        "env_id": config.env_id,
+        "model": config.model,
+        "num_examples": config.num_examples,
+        "rollouts_per_example": config.rollouts_per_example,
+        "shuffle": config.shuffle,
+        "shuffle_seed": config.shuffle_seed,
+        "n": agg.get("n"),
+        "avg_reward": agg.get("avg_reward"),
+        "avg_metrics": agg.get("metrics"),
+    }
+    save_metadata(results_path, metadata)
+    return metadata
+
+
+def _adapt_client(client: Any) -> Any:
+    """Bridge verifiers' client API drift. Older releases take a raw (Async)OpenAI
+    client; 0.1.14+ requires its own Client/ClientConfig ("Unsupported client
+    type"). Map a raw client's base_url onto a ClientConfig, keyed by the env
+    var that holds the matching API key (ClientConfig reads the key from the
+    environment by NAME)."""
+    try:
+        from verifiers.clients import Client  # noqa: WPS433 — new-style verifiers only
+        from verifiers.types import ClientConfig  # noqa: WPS433
+    except Exception:  # noqa: BLE001 — old verifiers: raw clients accepted
+        return client
+    if isinstance(client, (Client, ClientConfig)):
+        return client
+    base_url = str(getattr(client, "base_url", "") or "")
+    api_key_var = (
+        "OPENAI_API_KEY" if "api.openai.com" in base_url else "OPENROUTER_API_KEY"
+    )
+    return ClientConfig(
+        client_type="openai_chat_completions",
+        api_base_url=base_url or "https://openrouter.ai/api/v1",
+        api_key_var=api_key_var,
+    )
+
+
+def score_environment(
+    env: Any,
+    *,
+    client: Any,
+    model: str,
+    num_examples: int = -1,
+    rollouts_per_example: int = 1,
+    **kwargs: Any,
+) -> Any:
+    """Run a verifiers environment and return its ``GenerateOutputs``. `env` is an
+    already-loaded Environment, or an env id/name resolvable by
+    ``verifiers.load_environment`` (a local container must be installed first)."""
+    if isinstance(env, str):
+        try:
+            import verifiers as vf
+        except ImportError as e:  # pragma: no cover - optional dep
+            raise EvalHarvestError(
+                'verifiers is not installed — `pip install "verifiers<1.0.0"` to '
+                "score environments (this adapter drives the legacy v0 "
+                "Environment API, which v1 froze), or pass an already-loaded "
+                "Environment."
+            ) from e
+        if not hasattr(vf, "load_environment"):
+            # verifiers v1 dropped the frozen v0 loader — this adapter is v0-only.
+            raise EvalHarvestError(
+                "the installed verifiers has no load_environment — it looks like "
+                "verifiers v1, whose Taskset API this v0 adapter does not drive. "
+                'Pin `verifiers<1.0.0` for v0 environments, or run v1 Taskset '
+                "packages through the platform runner's verifiers-v1 engine."
+            )
+        env = vf.load_environment(env)
+    if not hasattr(env, "evaluate_sync"):
+        if callable(getattr(env, "load", None)):
+            # v1 Tasksets expose load() -> list[Task] instead of evaluate_sync.
+            raise EvalHarvestError(
+                "env looks like a verifiers v1 Taskset (has load(), no "
+                "evaluate_sync). This adapter drives the legacy v0 Environment "
+                "API only — run v1 Taskset packages through the platform "
+                "runner's verifiers-v1 engine, or pin `verifiers<1.0.0` and "
+                "expose a v0 load_environment()."
+            )
+        raise EvalHarvestError("env is not a verifiers Environment (no evaluate_sync method).")
+    return env.evaluate_sync(
+        client=_adapt_client(client),
+        model=model,
+        num_examples=num_examples,
+        rollouts_per_example=rollouts_per_example,
+        **kwargs,
+    )
+
+
+def run_and_harvest(
+    env: Any,
+    version_id: str,
+    evals_client: GatewayEvalsClient,
+    *,
+    client: Any,
+    model: str,
+    num_examples: int = -1,
+    rollouts_per_example: int = 1,
+    run_id: Optional[str] = None,
+    **kwargs: Any,
+) -> Dict[str, Any]:
+    """Score a verifiers container version and harvest a version-pinned evaluation
+    into Gateway: evaluate → create → push samples → finalize with metrics."""
+    outputs = score_environment(
+        env,
+        client=client,
+        model=model,
+        num_examples=num_examples,
+        rollouts_per_example=rollouts_per_example,
+        **kwargs,
+    )
+    samples = samples_from_generate_outputs(outputs)
+    metrics = metrics_from_generate_outputs(outputs)
+
+    evaluation_id = evals_client.create_evaluation(
+        version_id,
+        model=model,
+        run_id=run_id,
+        # Samples are one row PER ROLLOUT — count distinct examples, not rows,
+        # or the UI's "N examples × k rollouts" double-counts by k.
+        num_examples=(num_examples if num_examples > 0 else len({s.get("exampleId") for s in samples})),
+        rollouts_per_example=rollouts_per_example,
+    )
+    inserted = evals_client.push_samples(evaluation_id, samples)
+    evals_client.finalize_evaluation(evaluation_id, metrics=metrics)
+
+    return {
+        "evaluationId": evaluation_id,
+        "samples": inserted,
+        "metrics": metrics,
+    }
+
+
+def run_local_eval(
+    env: Any,
+    version_id: str,
+    evals_client: GatewayEvalsClient,
+    *,
+    client: Any,
+    config: Any,
+    run_id: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Run a local verifiers eval driven by an ``EvalConfig``, writing a resumable
+    on-disk artifact (``results.jsonl`` + ``metadata.json``) when
+    ``config.save_results`` and harvesting a version-pinned evaluation into Gateway.
+
+    Resume: ``config.resume_path`` uses that dir; otherwise, when saving, it reuses
+    the newest matching incomplete run (or starts a fresh one). NOTE: v1 saves the
+    whole run at the end (resume across runs), not mid-``evaluate_sync``.
+    """
+    from pathlib import Path
+
+    from .path_utils import (
+        find_latest_incomplete_eval_results_path,
+        get_results_path,
+    )
+
+    results_path = None
+    if config.resume_path:
+        results_path = Path(config.resume_path)
+    elif config.save_results:
+        results_path = find_latest_incomplete_eval_results_path(
+            config.output_dir,
+            config.env_id,
+            config.model,
+            config.num_examples,
+            config.rollouts_per_example,
+            config.shuffle,
+            config.shuffle_seed,
+        ) or get_results_path(config.output_dir, config.env_id, config.model)
+
+    outputs = score_environment(
+        env,
+        client=client,
+        model=config.model,
+        num_examples=config.num_examples,
+        rollouts_per_example=config.rollouts_per_example,
+    )
+
+    if results_path is not None:
+        write_results_artifact(outputs, config, results_path)
+
+    samples = samples_from_generate_outputs(outputs)
+    metrics = metrics_from_generate_outputs(outputs)
+    evaluation_id = evals_client.create_evaluation(
+        version_id,
+        model=config.model,
+        run_id=run_id,
+        # Distinct examples, not rollout rows (see run_and_harvest).
+        num_examples=(config.num_examples if config.num_examples > 0 else len({s.get("exampleId") for s in samples})),
+        rollouts_per_example=config.rollouts_per_example,
+    )
+    inserted = evals_client.push_samples(evaluation_id, samples)
+    evals_client.finalize_evaluation(evaluation_id, metrics=metrics)
+
+    return {
+        "evaluationId": evaluation_id,
+        "samples": inserted,
+        "metrics": metrics,
+        "resultsPath": str(results_path) if results_path else None,
+    }

gatewaysdk/benchmark_hub/versioning.py

@@ -0,0 +1,72 @@
+"""Semantic-version bump helpers for `bench push --auto-bump/--rc/--post`.
+
+The bump functions and the pyproject rewrite are ported verbatim from Prime
+Intellect's ``prime`` CLI (``packages/prime/src/prime_cli/commands/env.py``,
+MIT (c) 2025 Prime Intellect) — see ``_archive.py`` for the same attribution. The
+rewrite is a raw-text regex substitution (not a TOML re-dump) so comments and
+formatting in ``pyproject.toml`` are preserved.
+
+Stdlib only (``re``).
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+
+def bump_version(version: str) -> str:
+    """Patch bump: ``1.2.3`` -> ``1.2.4``. Strips a ``-``/``+`` pre-release/build
+    suffix to find the numeric patch; a non-numeric patch appends ``.1``."""
+    parts = version.split(".")
+    if len(parts) >= 3:
+        patch_part = parts[2]
+        if "-" in patch_part:
+            patch_num = patch_part.split("-")[0]
+        elif "+" in patch_part:
+            patch_num = patch_part.split("+")[0]
+        else:
+            patch_num = patch_part
+        try:
+            parts[2] = str(int(patch_num) + 1)
+            return ".".join(parts)
+        except ValueError:
+            return f"{version}.1"
+    elif len(parts) == 2:
+        return f"{version}.1"
+    else:
+        return f"{version}.0.1"
+
+
+def bump_rc_version(version: str) -> str:
+    """Release-candidate bump: ``1.2.3`` -> ``1.2.3.rc0``, ``1.2.3.rc0`` ->
+    ``1.2.3.rc1`` (Prime's docstring wrongly says ``.post``; the code does ``.rc``)."""
+    m = re.match(r"^(?P<base>.*?)(?:\.rc|rc)(?P<num>\d+)$", version)
+    if m:
+        return f"{m.group('base')}.rc{int(m.group('num')) + 1}"
+    return f'{re.sub(r"([+-].*)$", "", version)}.rc0'
+
+
+def bump_post_version(version: str) -> str:
+    """Post-release bump: ``1.2.3`` -> ``1.2.3.post0``, ``1.2.3.post0`` ->
+    ``1.2.3.post1``."""
+    m = re.match(r"^(?P<base>.*?)(?:\.post|post)(?P<num>\d+)$", version)
+    if m:
+        return f"{m.group('base')}.post{int(m.group('num')) + 1}"
+    return f'{re.sub(r"([+-].*)$", "", version)}.post0'
+
+
+def update_pyproject_version(pyproject_path: Path, new_version: str) -> None:
+    """Rewrite the ``version = "..."`` line in pyproject.toml via regex (preserving
+    formatting/comments). Raises ``ValueError`` if no version line is present."""
+    pyproject_path = Path(pyproject_path)
+    content = pyproject_path.read_text()
+    updated = re.sub(
+        r'(\s*)version\s*=\s*["\'][^"\']*["\']',
+        rf'\1version = "{new_version}"',
+        content,
+        flags=re.MULTILINE,
+    )
+    if updated == content:
+        raise ValueError("version line not found or unchanged in pyproject.toml")
+    pyproject_path.write_text(updated)