gatewaysdk 0.3.2__py3-none-any.whl

gatewaysdk/automations.py

@@ -0,0 +1,111 @@
+"""Automations client for Gateway public REST API."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Mapping, MutableMapping, Optional
+from urllib import parse
+
+import requests
+
+
+class AutomationClientError(RuntimeError):
+    """Raised when the Gateway automations API returns an error response."""
+
+    def __init__(self, message: str, *, status_code: Optional[int] = None, body: Optional[str] = None) -> None:
+        super().__init__(message)
+        self.status_code = status_code
+        self.body = body
+
+
+class AutomationClient:
+    """HTTP client for project-scoped automations endpoints (Basic auth)."""
+
+    def __init__(self, base_url: str, public_key: str, secret_key: str, *, timeout_seconds: float = 60.0) -> None:
+        self._base = base_url.rstrip("/")
+        self._timeout = timeout_seconds
+        self._session = requests.Session()
+        self._session.auth = (public_key, secret_key)
+        self._session.headers.update({"Accept": "application/json"})
+
+    def _request(
+        self,
+        method: str,
+        path: str,
+        *,
+        query: Optional[Mapping[str, Any]] = None,
+        json_body: Optional[MutableMapping[str, Any]] = None,
+    ) -> Any:
+        url = f"{self._base}{path}"
+        params = {k: v for k, v in (query or {}).items() if v is not None}
+        try:
+            resp = self._session.request(
+                method,
+                url,
+                params=params or None,
+                json=json_body,
+                timeout=self._timeout,
+            )
+        except requests.RequestException as exc:
+            raise AutomationClientError(f"Automations HTTP request failed for {path}: {exc}") from exc
+
+        try:
+            out = resp.json() if resp.content else {}
+        except ValueError:
+            out = {}
+
+        if not resp.ok:
+            msg = str(out.get("message", resp.reason or resp.text))
+            raise AutomationClientError(
+                f"Gateway automations API error {resp.status_code} on {path}: {msg}",
+                status_code=resp.status_code,
+                body=resp.text,
+            )
+        return out
+
+    def list(self) -> List[Dict[str, Any]]:
+        """GET /api/public/automations — list automations for the authenticated project."""
+        out = self._request("GET", "/api/public/automations")
+        if not isinstance(out, dict):
+            raise AutomationClientError(f"Unexpected list() response shape: {type(out).__name__}")
+        data = out.get("automations")
+        if not isinstance(data, list):
+            raise AutomationClientError("Response missing 'automations' array")
+        return [dict(x) for x in data if isinstance(x, dict)]
+
+    def create(
+        self,
+        name: str,
+        trigger_config: Mapping[str, Any],
+        action_config: Mapping[str, Any],
+    ) -> Dict[str, Any]:
+        """POST /api/public/automations — create trigger, action, and automation."""
+        body = {"name": name, "trigger": dict(trigger_config), "action": dict(action_config)}
+        out = self._request("POST", "/api/public/automations", json_body=body)
+        if not isinstance(out, dict):
+            raise AutomationClientError(f"Unexpected create() response shape: {type(out).__name__}")
+        return dict(out)
+
+    def trigger(self, automation_id: str, payload: Optional[Mapping[str, Any]] = None) -> Dict[str, Any]:
+        """POST /api/public/automations/{id}/trigger — enqueue a manual run; optional JSON body becomes execution input."""
+        path = f"/api/public/automations/{parse.quote(automation_id, safe='')}/trigger"
+        out = self._request("POST", path, json_body=dict(payload) if payload is not None else {})
+        if not isinstance(out, dict):
+            raise AutomationClientError(f"Unexpected trigger() response shape: {type(out).__name__}")
+        return dict(out)
+
+    def get_executions(
+        self,
+        automation_id: str,
+        *,
+        limit: int = 50,
+        offset: int = 0,
+    ) -> List[Dict[str, Any]]:
+        """GET /api/public/automations/{id}/executions — paginated execution history."""
+        path = f"/api/public/automations/{parse.quote(automation_id, safe='')}/executions"
+        out = self._request("GET", path, query={"limit": limit, "offset": offset})
+        if not isinstance(out, dict):
+            raise AutomationClientError(f"Unexpected get_executions() response shape: {type(out).__name__}")
+        data = out.get("executions")
+        if not isinstance(data, list):
+            raise AutomationClientError("Response missing 'executions' array")
+        return [dict(x) for x in data if isinstance(x, dict)]

gatewaysdk/benchmark_hub/README.md

@@ -0,0 +1,88 @@
+# benchmark_hub — Gateway benchmark container hub SDK
+
+Push, version, browse, and evaluate benchmark **containers** (verifiers-format
+environments) on Gateway. This is Prime Intellect's hub model ported onto the
+Gateway stack: the **source bundle is the versioned, browsable source of truth**;
+an OCI image is built per version for execution.
+
+Everything here talks **only to a Gateway host** — never to `app.primeintellect.ai`.
+
+## Auth + config (env)
+
+| Env var | Purpose |
+|---|---|
+| `GATEWAY_HOST` | Gateway base URL, e.g. `https://withgateway.ai` |
+| `GATEWAY_PUBLIC_KEY` / `GATEWAY_SECRET_KEY` | project API key pair — **writes require the secret key** (Basic auth) |
+
+Bundle storage is flexible/enterprise-overridable server-side via
+`LANGFUSE_S3_BENCHMARK_UPLOAD_*` (falls back to the media-upload bucket under a
+`benchmark-containers/` prefix — zero-config by default).
+
+## Push a container
+
+```python
+from gatewaysdk.benchmark_hub import GatewayBenchmarkClient
+
+client = GatewayBenchmarkClient.from_env()           # GATEWAY_HOST/PUBLIC_KEY/SECRET_KEY
+result = client.push("./automationbench", visibility="PUBLIC")
+# resolve -> versions (presigned PUT) -> finalize
+print(result)  # {containerId, versionId, contentHash, state, slug, alreadyUploaded}
+```
+
+CLI: `python -m gatewaysdk.benchmark_hub.client push ./automationbench --visibility PUBLIC`
+
+The push computes a **source-only `content_hash`** (vendored byte-compatible with
+Prime's hub) as the logical version id, builds a deterministic source tarball,
+and sends the tarball's real sha256 as the S3 checksum (distinct from the content
+hash — this is what makes the presigned upload succeed). The **full file tree** is
+recorded separately for the browser.
+
+## Harvest a version-pinned evaluation
+
+```python
+from gatewaysdk.benchmark_hub import GatewayEvalsClient
+
+evals = GatewayEvalsClient.from_env()
+eval_id = evals.create_evaluation(version_id, model="gpt-4o-mini")
+evals.push_samples(eval_id, samples)                 # batched; multi-reward per sample
+evals.finalize_evaluation(eval_id, metrics={"pass_rate": 0.87}, cost={"total_usd": 0.42})
+```
+
+An evaluation is **pinned to the immutable container version**, so scores stay
+reproducible and diffable across re-pushes. Each sample carries the overall score
+**and** the named sub-reward breakdown (`rewards`), never just an average.
+`normalize_sample` accepts prime-evals snake_case (`example_id`, `rollout_index`)
+so a verifiers/prime `Sample` harvests unchanged.
+
+## Server API (under `/api/public/benchmark-containers`)
+
+| Route | Auth | Purpose |
+|---|---|---|
+| `POST /resolve` | write | get-or-create a container |
+| `POST /versions` | write | create/dedup a version + presigned upload |
+| `POST /versions/{id}/finalize` | write | mark the source READY |
+| `GET  /` | read | list containers + versions |
+| `GET  /versions/{id}/files` | read | full file tree (browser) |
+| `GET  /versions/{id}/download` | read | signed URL to the source tarball |
+| `POST /evaluations` | write | create a version-pinned evaluation |
+| `POST /evaluations/{id}/samples` | write | append multi-reward sample batches |
+| `POST /evaluations/{id}/finalize` | write | metrics + cost + status |
+
+## What's vendored vs ours (per the integration strategy)
+
+- **Vendored (MIT, Prime):** the four content-hash helpers in `_archive.py` —
+  byte-compatible hashing for hub interop. There is no library to wrap; the rest
+  of Prime's hub is a CLI welded to Typer/Rich.
+- **Ours:** the hub backend (schema, routes, storage), the reimplemented push +
+  eval clients (Prime's `prime-sandboxes`/`prime env`/hosted eval all terminate
+  at `api.primeintellect.ai` and aren't self-hostable).
+- **Borrowed shape:** the eval client mirrors prime-evals' create→push→finalize,
+  harvesting into Gateway's own store.
+
+## Not yet wired
+
+- **`verifiers` scoring** — for verifiers-format containers, `Environment.evaluate()`
+  produces the sample rows for free (the one genuinely reusable open Prime lib);
+  the worker that runs a finalized container should feed the eval routes from it.
+- **Per-version OCI image build** — `content_hash`-tagged image so the runner can
+  execute the container (`generic --image`); `image_ref` stays null until then.

gatewaysdk/benchmark_hub/__init__.py

@@ -0,0 +1,119 @@
+"""Gateway benchmark-container hub SDK.
+
+Client-side helpers for packaging a benchmark "container" (a verifiers-format
+environment: ``pyproject.toml`` + source modules + README) and pushing it to the
+Gateway hub for content-addressed version control.
+
+The content-hash helpers in :mod:`._archive` are vendored verbatim from Prime
+Intellect's ``prime`` CLI (MIT) so our ``content_hash`` is byte-compatible with
+theirs — see that module's header for attribution.
+"""
+
+from ._archive import collect_archive_files, compute_content_hash
+from ._tar import safe_tar_extract
+from .client import (
+    BenchmarkPushError,
+    DEFAULT_BRANCH,
+    GatewayBenchmarkClient,
+    build_file_manifest,
+    build_source_tarball,
+    build_wheel,
+    normalize_package_name,
+    parse_wheel_requires_dist,
+    local_tree_state,
+    read_local_meta,
+    read_pyproject,
+    sha256_base64,
+    write_local_meta,
+)
+from .eval_config import EvalConfig
+from .evals import (
+    EvalHarvestError,
+    GatewayEvalsClient,
+    normalize_sample,
+)
+from .path_utils import (
+    count_saved_rollouts,
+    find_latest_incomplete_eval_results_path,
+    get_results_path,
+    is_valid_eval_results_path,
+)
+from .save_utils import (
+    load_metadata,
+    load_outputs,
+    save_metadata,
+    save_outputs,
+    validate_resume_metadata,
+)
+from .harbor_adapter import (
+    detect_env_kind,
+    find_mcp_config,
+    harvest_job_dir,
+    resolve_harbor_model,
+    run_harbor_job,
+    samples_from_job_dir,
+    tasks_root,
+)
+from .verifiers_adapter import (
+    metrics_from_generate_outputs,
+    run_and_harvest,
+    run_local_eval,
+    samples_from_generate_outputs,
+    score_environment,
+    write_results_artifact,
+)
+from .versioning import (
+    bump_post_version,
+    bump_rc_version,
+    bump_version,
+    update_pyproject_version,
+)
+
+__all__ = [
+    "compute_content_hash",
+    "collect_archive_files",
+    "safe_tar_extract",
+    "bump_version",
+    "bump_rc_version",
+    "bump_post_version",
+    "update_pyproject_version",
+    "GatewayBenchmarkClient",
+    "DEFAULT_BRANCH",
+    "local_tree_state",
+    "read_local_meta",
+    "write_local_meta",
+    "BenchmarkPushError",
+    "build_source_tarball",
+    "build_file_manifest",
+    "build_wheel",
+    "parse_wheel_requires_dist",
+    "normalize_package_name",
+    "sha256_base64",
+    "read_pyproject",
+    "GatewayEvalsClient",
+    "EvalHarvestError",
+    "normalize_sample",
+    "samples_from_generate_outputs",
+    "metrics_from_generate_outputs",
+    "score_environment",
+    "run_and_harvest",
+    "run_local_eval",
+    "write_results_artifact",
+    "detect_env_kind",
+    "tasks_root",
+    "run_harbor_job",
+    "harvest_job_dir",
+    "samples_from_job_dir",
+    "find_mcp_config",
+    "resolve_harbor_model",
+    "EvalConfig",
+    "save_outputs",
+    "load_outputs",
+    "save_metadata",
+    "load_metadata",
+    "validate_resume_metadata",
+    "get_results_path",
+    "is_valid_eval_results_path",
+    "count_saved_rollouts",
+    "find_latest_incomplete_eval_results_path",
+]

gatewaysdk/benchmark_hub/_archive.py

@@ -0,0 +1,189 @@
+"""Deterministic content hashing for benchmark-container source bundles.
+
+Vendored verbatim (with only docstring/typing cosmetics) from Prime Intellect's
+``prime`` CLI so that the ``content_hash`` we compute for a container is
+byte-for-byte identical to the one their Environments Hub computes for the same
+files. That byte-compatibility is the whole point of copying rather than
+reimplementing: a container hashed here and on hub.primeintellect.ai resolves to
+the same logical version id.
+
+Source: https://github.com/PrimeIntellect-ai/prime
+        packages/prime/src/prime_cli/commands/env.py  (functions at L575-L689)
+License: MIT (c) Prime Intellect. See THIRD_PARTY_LICENSES for the full text.
+
+IMPORTANT: this is the *source-only* collection. Prime-compatible bundles hash
+only ``README.md``, ``pyproject.toml`` and ``*.py`` at the root (plus nested
+package dirs). Gateway-only bundles hash every non-hidden root file as well:
+mock-tools bundles (root ``gateway-env.toml``) carry their executable contract
+in root ``tasks.jsonl`` and ``tool-schemas.json``, and api worlds (root
+``gateway-api-world.toml``) carry theirs in the marker itself plus root
+``openapi.json`` — omitting any of these produced a READY bundle the platform
+could not detect or execute. Dotfiles, ``__pycache__``, build outputs and
+``.gitignore``-d paths are still dropped.
+The hash is the version primary key. The *full* file tree shown in the
+GitHub-style browser is collected and stored separately and must NOT feed this
+function, or hashes would diverge from Prime's.
+"""
+
+import hashlib
+import os
+from pathlib import Path
+from typing import Callable, Dict, List, Optional
+
+try:  # optional: only needed when a container actually ships a .gitignore
+    from gitignore_parser import parse_gitignore
+except ImportError:  # pragma: no cover - exercised only when the dep is absent
+    parse_gitignore = None  # type: ignore[assignment]
+
+
+def should_include_file_in_archive(file_path: Path, base_path: Path) -> bool:
+    """Return True if ``file_path`` belongs in the hashed source archive."""
+    if not file_path.is_file():
+        return False
+
+    # Skip symlinks - they cause extraction failures on the server side.
+    if file_path.is_symlink():
+        return False
+
+    rel_path = file_path.relative_to(base_path)
+
+    # Skip hidden files.
+    if file_path.name.startswith("."):
+        return False
+
+    # Skip files in __pycache__ directories.
+    if "__pycache__" in rel_path.parts:
+        return False
+
+    return True
+
+
+def should_include_directory_in_archive(dir_path: Path) -> bool:
+    """Return True if ``dir_path`` should be traversed into the archive."""
+    if not dir_path.is_dir():
+        return False
+
+    # Skip hidden directories (includes .prime/, .git/, etc.).
+    if dir_path.name.startswith("."):
+        return False
+
+    # Skip build artifacts, cache directories, and outputs.
+    if dir_path.name in ["dist", "__pycache__", "build", "outputs"]:
+        return False
+
+    # Skip egg-info directories.
+    if dir_path.name.endswith(".egg-info"):
+        return False
+
+    return True
+
+
+def _build_gitignore_matcher(env_path: Path) -> Optional[Callable[[str], bool]]:
+    """Build a matcher for the root .gitignore file, if present."""
+    gitignore_path = env_path / ".gitignore"
+    if not gitignore_path.exists():
+        return None
+    if parse_gitignore is None:
+        # Explicit failure rather than silently producing a hash that would
+        # diverge from Prime's by including .gitignore-d files.
+        raise RuntimeError(
+            "A .gitignore is present but 'gitignore_parser' is not installed. "
+            "Install it (it is a declared dependency of gatewaysdk) so the "
+            "content hash stays compatible with the hub."
+        )
+    return parse_gitignore(str(gitignore_path), base_dir=str(env_path))
+
+
+def collect_archive_files(env_path: Path) -> List[Path]:
+    """Collect archive file paths in deterministic order, honoring .gitignore."""
+    env_path = Path(env_path)  # accept str paths (CLI passes a string)
+    ignore_matcher = _build_gitignore_matcher(env_path)
+    files_by_rel_path: Dict[str, Path] = {}
+
+    def maybe_add_file(file_path: Path) -> None:
+        if not should_include_file_in_archive(file_path, env_path):
+            return
+        if ignore_matcher is not None and ignore_matcher(str(file_path)):
+            return
+
+        rel_path = str(file_path.relative_to(env_path)).replace("\\", "/")
+        files_by_rel_path[rel_path] = file_path
+
+    # Prime bundles keep Prime's root globs. A Gateway-only root marker widens
+    # them to every ordinary root file, because those kinds keep executable
+    # contract at the root: mock-tools (``gateway-env.toml``) in ``tasks.jsonl``
+    # / ``tool-schemas.json``, api worlds (``gateway-api-world.toml``) in the
+    # marker itself and ``openapi.json``. Include them so the archive and its
+    # content hash describe the complete runnable source rather than a
+    # READY-but-broken subset — an api world whose marker never reached the
+    # bundle was detected as harbor and surfaced zero tools.
+    #
+    # The manifest MUST be hashed or a manifest-only edit — changing the served
+    # tool list, the kind, the db, or the dependency pins — produces an identical
+    # hash, dedups into the previous version, and is silently discarded while
+    # reporting success. That made every such edit a no-op.
+    #
+    # This does not break Prime compatibility: a Prime-compatible bundle has no
+    # Gateway marker, so it continues to use the original globs unchanged.
+    root_patterns = ["README.md", "pyproject.toml", "*.py"]
+    if (env_path / "gateway-env.toml").is_file() or (
+        env_path / "gateway-api-world.toml"
+    ).is_file():
+        root_patterns.append("*")
+    for pattern in root_patterns:
+        for file_path in sorted(env_path.glob(pattern), key=lambda p: p.name):
+            maybe_add_file(file_path)
+
+    def is_nested_dir_ignored(dir_path: Path) -> bool:
+        """Check if a nested directory should be pruned from traversal."""
+        if not should_include_directory_in_archive(dir_path):
+            return True
+        if ignore_matcher is not None and ignore_matcher(str(dir_path)):
+            return True
+        return False
+
+    for subdir in sorted(env_path.iterdir(), key=lambda path: path.name):
+        if not should_include_directory_in_archive(subdir):
+            continue
+        if is_nested_dir_ignored(subdir):
+            continue
+
+        for root, dirnames, filenames in os.walk(subdir):
+            root_path = Path(root)
+            dirnames[:] = sorted(
+                dirname for dirname in dirnames if not is_nested_dir_ignored(root_path / dirname)
+            )
+            for filename in sorted(filenames):
+                maybe_add_file(root_path / filename)
+
+    return [files_by_rel_path[rel_path] for rel_path in sorted(files_by_rel_path)]
+
+
+def compute_content_hash(env_path: Path) -> str:
+    """Compute a deterministic, cross-platform content hash for a container.
+
+    SHA-256 over each included file's normalized relative path (prefixed with
+    ``file:``) followed by its raw bytes, in sorted path order. This is a
+    *logical* digest of the source set — NOT the sha256 of any tarball object,
+    so it cannot double as an S3 upload checksum.
+
+    Args:
+        env_path: Path to the container directory.
+
+    Returns:
+        SHA-256 hexdigest of the container content.
+    """
+    env_path = Path(env_path)  # accept str paths (CLI passes a string)
+    content_hasher = hashlib.sha256()
+
+    for file_path in collect_archive_files(env_path):
+        normalized_path = str(file_path.relative_to(env_path)).replace("\\", "/")
+        content_hasher.update(f"file:{normalized_path}".encode("utf-8"))
+        try:
+            with open(file_path, "rb") as f:
+                content_hasher.update(f.read())
+        except IOError:
+            # Skip files that can't be read.
+            pass
+
+    return content_hasher.hexdigest()

gatewaysdk/benchmark_hub/_tar.py

@@ -0,0 +1,6 @@
+"""Guarded tar extraction; the implementation lives in ``gatewaysdk.environment._tar``
+so the World Host image, which ships the environment package alone, has it too."""
+
+from ..environment._tar import safe_tar_extract
+
+__all__ = ["safe_tar_extract"]