benchmark-router-cli 0.1.0__tar.gz

benchmark_router_cli-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+node_modules/
+.next/
+.env*
+!.env.example
+__pycache__/
+*.pyc
+.venv/
+.vercel/
+.DS_Store
+.claude/

benchmark_router_cli-0.1.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.5
+Name: benchmark-router-cli
+Version: 0.1.0
+Summary: Command-line client for Benchmark Router
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: pyyaml>=6
+Requires-Dist: rich>=13
+Requires-Dist: typer>=0.12

benchmark_router_cli-0.1.0/brouter_cli/client.py

@@ -0,0 +1,95 @@
+from typing import Any
+
+import httpx
+
+from .config import Settings
+
+
+class ApiError(Exception):
+    def __init__(self, status: int, message: str):
+        super().__init__(f"{status}: {message}")
+        self.status = status
+        self.message = message
+
+
+class Client:
+    def __init__(self, settings: Settings, transport: httpx.BaseTransport | None = None):
+        if not settings.api_key:
+            raise ApiError(401, "no API key; run `brouter login` or set BROUTER_API_KEY")
+        self.api_url = settings.api_url
+        self._http = httpx.Client(
+            base_url=f"{settings.api_url}/api/v1",
+            headers={"Authorization": f"Bearer {settings.api_key}"},
+            timeout=60,
+            transport=transport,
+        )
+
+    def _request(self, method: str, path: str, **kwargs: Any) -> Any:
+        resp = self._http.request(method, path, **kwargs)
+        if resp.status_code >= 400:
+            try:
+                body = resp.json()
+                message = body.get("error") or body.get("message") or resp.text
+            except ValueError:
+                message = resp.text
+            raise ApiError(resp.status_code, str(message))
+        return resp.json()
+
+    def benchmarks(self) -> list[dict]:
+        return self._request("GET", "/benchmarks")["benchmarks"]
+
+    def models(self) -> list[dict]:
+        return self._request("GET", "/models")["models"]
+
+    def estimate(self, body: dict) -> dict:
+        return self._request("POST", "/estimate", json=body)
+
+    def create_run(self, body: dict) -> dict:
+        return self._request("POST", "/runs", json=body)["run"]
+
+    def runs(self) -> list[dict]:
+        return self._request("GET", "/runs")["runs"]
+
+    def run(self, run_id: str) -> dict:
+        return self._request("GET", f"/runs/{run_id}")["run"]
+
+    def download(self, run_id: str) -> list[dict]:
+        return self._request("GET", f"/runs/{run_id}/download")["files"]
+
+    def balance(self) -> dict:
+        return self._request("GET", "/balance")
+
+    def task(self, run_id: str) -> dict:
+        return self._request("GET", f"/runs/{run_id}/task")
+
+    def comparisons(self) -> list[dict]:
+        return self._request("GET", "/comparisons")["comparisons"]
+
+    def estimate_comparison(self, harness_id: str, model_ids: list[str]) -> dict:
+        return self._request("POST", "/comparisons", json={"harness_id": harness_id, "model_ids": model_ids, "dry_run": True})
+
+    def create_comparison(self, harness_id: str, model_ids: list[str]) -> dict:
+        # The server submits one run per benchmark x entrant, so this can take minutes.
+        body = {"harness_id": harness_id, "model_ids": model_ids}
+        return self._request("POST", "/comparisons", json=body, timeout=300)["comparison"]
+
+    def update_comparison(self, comparison_id: str, body: dict) -> dict:
+        return self._request("PATCH", f"/comparisons/{comparison_id}", json=body)["comparison"]
+
+    def harness_requests(self) -> list[dict]:
+        return self._request("GET", "/harness-requests")["requests"]
+
+    def create_harness_request(self, body: dict) -> dict:
+        return self._request("POST", "/harness-requests", json=body)["request"]
+
+    def benchmark_requests(self) -> list[dict]:
+        return self._request("GET", "/benchmark-requests")["requests"]
+
+    def create_benchmark_request(self, body: dict) -> dict:
+        return self._request("POST", "/benchmark-requests", json=body)["request"]
+
+    def checkout(self, amount: float) -> str:
+        return self._request("POST", "/credits/checkout", json={"credits": amount})["url"]
+
+    def fetch(self, url: str) -> httpx.Response:
+        return httpx.get(url, timeout=600, follow_redirects=True)

benchmark_router_cli-0.1.0/brouter_cli/config.py

@@ -0,0 +1,50 @@
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+DEFAULT_API_URL = "https://benchmarkrouter.newmeasure.ai"
+
+
+def _config_dir() -> Path:
+    return Path(os.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config")
+
+
+def config_path() -> Path:
+    return _config_dir() / "brouter" / "config.json"
+
+
+def _env(name: str) -> str | None:
+    # EVALHUB_* is the pre-rename spelling, still honoured.
+    return os.environ.get(f"BROUTER_{name}") or os.environ.get(f"EVALHUB_{name}")
+
+
+@dataclass
+class Settings:
+    api_url: str
+    api_key: str | None
+
+
+def load_settings() -> Settings:
+    stored: dict = {}
+    path = config_path()
+    legacy = _config_dir() / "evalh" / "config.json"
+    if not path.exists() and legacy.exists():
+        path = legacy
+    if path.exists():
+        stored = json.loads(path.read_text())
+    return Settings(
+        api_url=(_env("API_URL") or stored.get("api_url") or DEFAULT_API_URL).rstrip("/"),
+        api_key=_env("API_KEY") or stored.get("api_key"),
+    )
+
+
+def save_settings(api_key: str, api_url: str | None) -> Path:
+    path = config_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    data = {"api_key": api_key}
+    if api_url:
+        data["api_url"] = api_url.rstrip("/")
+    path.write_text(json.dumps(data, indent=2))
+    path.chmod(0o600)
+    return path