openrunner-sdk 2.37.4__tar.gz → 2.37.6__tar.gz

{openrunner_sdk-2.37.4 → openrunner_sdk-2.37.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openrunner-sdk
-Version: 2.37.4
+Version: 2.37.6
 Summary: OpenRunner SDK - W&B-compatible ML experiment tracking client
 Project-URL: Homepage, https://github.com/jqueguiner/openrunner
 Project-URL: Repository, https://github.com/jqueguiner/openrunner

{openrunner_sdk-2.37.4 → openrunner_sdk-2.37.6}/openrunner/__init__.py

@@ -181,7 +181,7 @@ launch.from_run = _launch_from_run  # type: ignore[attr-defined]
 # openrunner.trace.patch_openai() syntax
 trace.patch_openai = _patch_openai  # type: ignore[attr-defined]
 
-__version__ = "2.37.4"
+__version__ = "2.37.6"
 
 logger = logging.getLogger("openrunner")
 

{openrunner_sdk-2.37.4 → openrunner_sdk-2.37.6}/openrunner/install_commands.py

@@ -442,7 +442,7 @@ allowed-tools:
 Upgrade openrunner-sdk and reinstall slash commands:
 
 ```bash
-pip install --upgrade openrunner-sdk 2>&1 | tail -3 && python3 -c "
+pip install --upgrade --no-cache-dir openrunner-sdk 2>&1 | tail -3 && python3 -c "
 import sys, os
 from pathlib import Path
 for p in ([os.path.join(d, 'site-packages') for d in __import__('glob').glob(os.path.expanduser('~/.local/lib/python3.*/')) + __import__('glob').glob(os.path.expanduser('~/Library/Python/3.*/lib/python/'))]):
@@ -2596,6 +2596,256 @@ pulls** -- `openrunner gpu run --image ...` resolves the org's stored registry.)
 """
 
 
+DRIFT_CMD = """---
+name: {prefix}drift
+description: Instrument an inference API/service to report per-prediction events to OpenRunner inference-drift monitoring (distribution shift / PSI), segment by metadata, then watch + alert -- no raw payloads stored, only the metrics you choose
+argument-hint: "[project org/name]"
+allowed-tools:
+  - Bash
+  - Read
+  - Edit
+  - AskUserQuestion
+---
+
+Add **inference-drift monitoring** to an API/service: after each prediction,
+report a lightweight event (input features + output metrics) to OpenRunner, which
+tracks distribution shift (PSI), per-segment health and A/B. Only the numbers +
+tags you send are stored -- never the raw inputs/outputs.
+
+## The event shape (one per inference)
+
+- `inference_id` -- your request/prediction id (str).
+- `metadata` -- dict of scalars OR lists (the axes you'll slice by), e.g.
+  `{{"model": "v3", "lang": ["fr", "en"], "route": "asr", "region": "eu"}}`.
+- `metrics` -- dict of floats to watch, e.g.
+  `{{"confidence": 0.92, "latency_ms": 180, "tokens": 57, "input_len": 12.4}}`.
+- `monitor` (optional) -- the drift stream name; use one per model/endpoint.
+- `event_time` (optional) -- ISO timestamp; defaults to now.
+
+Choose metrics that reveal drift: output confidence/score, input length / feature
+stats, latency, class distribution. Metadata = segments to localise a drift.
+
+## Step 1 -- resolve the project id
+
+```bash
+openrunner login
+python3 -c "
+from openrunner.api_client import APIClient
+import json, os
+c = APIClient(base_url=os.environ.get('OPENRUNNER_BASE_URL','https://openrun.gladia.io'),
+              api_key=os.environ['OPENRUNNER_API_KEY'])
+print([(p['name'], p['id']) for p in c.list_projects()])
+"
+```
+
+## Step 2 -- instrument (NON-BLOCKING) -- FastAPI example
+
+Report AFTER responding, batched on a background thread, so you never add latency
+to the request. The ingest endpoint answers **429 + Retry-After** under load (the
+client backs off) -- so drop events rather than block the request path.
+
+```python
+import os, threading, queue, atexit
+from openrunner.api_client import APIClient
+
+PROJECT_ID = "..."   # from step 1
+_client = APIClient(base_url=os.environ.get("OPENRUNNER_BASE_URL", "https://openrun.gladia.io"),
+                    api_key=os.environ["OPENRUNNER_API_KEY"])
+_q: queue.Queue = queue.Queue(maxsize=10000)
+
+def _flush_loop():
+    buf = []
+    while True:
+        ev = _q.get()
+        if ev is not None:
+            buf.append(ev)
+        if buf and (ev is None or len(buf) >= 200):
+            _client.post_inference_events(PROJECT_ID, buf, monitor="asr-prod")
+            buf = []
+        if ev is None:
+            return
+
+threading.Thread(target=_flush_loop, daemon=True).start()
+atexit.register(lambda: _q.put(None))
+
+def record_inference(req_id, metadata, metrics):
+    try:
+        _q.put_nowait({{"inference_id": req_id, "metadata": metadata, "metrics": metrics}})
+    except queue.Full:
+        pass  # never block or fail the request because of monitoring
+
+# ... in your endpoint, AFTER producing the result:
+record_inference(
+    req_id,
+    {{"model": "v3", "lang": lang, "route": "asr"}},
+    {{"confidence": conf, "latency_ms": dt_ms, "audio_s": duration}},
+)
+```
+
+**Non-Python service?** POST the events directly:
+
+```
+POST /api/v1/projects/PROJECT_ID/inference-drift/ingest
+Authorization: Bearer <API_KEY>
+{{ "monitor": "asr-prod",
+   "events": [ {{"inference_id": "r1", "metadata": {{"lang": "fr"}}, "metrics": {{"confidence": 0.9}}}} ] }}
+-> {{"ingested": n}}   (429 + Retry-After under load -> back off)
+```
+
+**Batch offline** from a JSON file instead of instrumenting:
+
+```bash
+openrunner drift report events.json --project ORG/NAME --monitor asr-prod
+```
+
+## Step 3 -- watch + alert
+
+- UI: the **Drift** menu -> overview / timeseries / per-segment PSI / A/B.
+- CLI:
+  - `openrunner drift monitors --project ORG/NAME`
+  - `openrunner drift check --project ORG/NAME --metric confidence` (drift verdict)
+  - `openrunner drift ab --project ORG/NAME --metric confidence --variant-a v2 --variant-b v3`
+  - `openrunner drift ask --project ORG/NAME "did latency drift this week?"`
+- MCP (if the OpenRunner server is registered): the drift tools answer the same in-session.
+
+## Notes
+
+- **Nothing raw is stored** -- only the metrics + metadata you send.
+- One `monitor` per model/endpoint; segment with metadata (lang / region / route /
+  model version) so a drift can be traced to a slice.
+- **Sample** at high QPS (report 1 in N) -- PSI needs volume, not every event.
+- Old days age out to S3 parquet automatically (a hot window stays in Postgres).
+"""
+
+
+DRIFT_INSTALL_CMD = """---
+name: {prefix}drift-install
+description: Actually wire inference-drift monitoring INTO the API in THIS repo -- detect the framework + endpoints, add a non-blocking reporter, insert record_inference() after each prediction, add the dependency, and verify with a live event (vs /openrunner:drift which only explains it)
+argument-hint: "[project org/name] [--endpoint PATH] [--sample N]"
+allowed-tools:
+  - Bash
+  - Read
+  - Edit
+  - Write
+  - Grep
+  - Glob
+  - AskUserQuestion
+---
+
+Instrument the inference API in the **current repo** with OpenRunner drift
+monitoring -- edit the code, don't just describe it (see `/openrunner:drift` for
+the reference). Do each step; ask only when genuinely ambiguous.
+
+## Step 1 -- detect the API + inference endpoint(s)
+- Identify the framework (grep `fastapi`, `flask`, `django`, `express`,
+  `@app.post`, `APIRouter`, route decorators) and the file(s) with the
+  **inference** endpoint(s) -- the handlers that return model predictions.
+- If several, ask which to instrument (default: the prediction routes). Honour
+  `--endpoint PATH` when given.
+
+## Step 2 -- resolve project + creds
+- `openrunner login` if needed. Resolve the project id (from the `org/name` arg,
+  else `APIClient.list_projects()`; ask if ambiguous).
+- Creds are read from the service env at runtime: `OPENRUNNER_API_KEY`
+  (+ `OPENRUNNER_BASE_URL`). Confirm they are set in the deployment env; **never
+  hardcode the key**. Store `PROJECT_ID` as a module constant or env var.
+
+## Step 3 -- add the non-blocking reporter module
+Create `openrunner_drift.py` (match the repo layout) with a background batcher
+(thread + `queue`, drop on full, 200/flush, 429 backoff via the SDK) exposing
+`record_inference(inference_id, metadata, metrics)` -- the pattern from
+`/openrunner:drift`. For an async framework, use an asyncio task + `asyncio.Queue`
+instead of a thread.
+
+## Step 4 -- instrument each endpoint
+AFTER the response is produced, insert a wrapped `record_inference(...)`:
+- `inference_id` = the request id / a `uuid4().hex`.
+- `metadata` = input axes present in the handler: model version, language, route,
+  region, tenant/customer.
+- `metrics` = outputs + **latency measured around the model call** (wrap the
+  predict call with a monotonic timer), output confidence/score, input length,
+  token count.
+- Wrap in try/except (or the queue put_nowait) so monitoring **never** breaks or
+  slows the request. Apply `--sample N` (report 1 in N) if given.
+
+## Step 5 -- dependency + verify
+- Add `openrunner-sdk` to `requirements.txt` / `pyproject.toml` if missing.
+- Import-check the new module; send ONE synthetic event
+  (`post_inference_events(PROJECT_ID, [{{...}}])`) and confirm `ingested: 1`.
+- Point the user at the **Drift** UI and `openrunner drift check --project ORG/NAME
+  --metric confidence`.
+
+## Guardrails
+- NEVER block or fail the request path for monitoring (drop on queue-full).
+- NEVER store raw inputs/outputs -- only the chosen metrics + metadata.
+- NEVER hardcode the API key -- read from env.
+- Keep the diff minimal + match the repo's style/imports.
+"""
+
+
+BUILD_API_CMD = """---
+name: {prefix}build-api
+description: Scaffold a production inference API for a model in THIS repo -- FastAPI /predict + /health, typed request/response, Dockerfile (pushable via the OpenRunner registry), env config, and optional baked-in drift monitoring; ready to deploy on governed GPUs
+argument-hint: "[--model PATH_OR_ARTIFACT] [--framework fastapi] [--drift] [--registry NAME/ns]"
+allowed-tools:
+  - Bash
+  - Read
+  - Edit
+  - Write
+  - Grep
+  - Glob
+  - AskUserQuestion
+---
+
+Scaffold a serving API for a model in the **current repo** -- a real, runnable
+skeleton, not a description. Ask only what you can't infer.
+
+## Step 1 -- understand the model
+- Find the model: a local checkpoint, an `openrunner.use_model(...)` artifact, a
+  HF id, or a `predict()` the user points at (`--model`). Ask if unclear.
+- Determine input -> output shape (text/audio/image in; label/score/text out) to
+  type the request/response.
+
+## Step 2 -- scaffold the API (default FastAPI)
+Write a minimal, production-shaped service:
+- `app/main.py` -- FastAPI app; load the model ONCE at startup (lifespan), a typed
+  `POST /predict` (pydantic request/response), `GET /health` (model-loaded probe),
+  `GET /` version. Batch-friendly handler; no blocking work in the event loop for
+  heavy models (run in a threadpool / worker).
+- `app/model.py` -- load + `predict()` wrapper (pull weights via
+  `openrunner.use_model` when the model is an OpenRunner artifact).
+- `requirements.txt` / `pyproject.toml`, `.env.example` (PORT, model ref,
+  `OPENRUNNER_*`), `README.md` (run + curl example).
+- `Dockerfile` -- slim base, install deps, copy app, `CMD` uvicorn on `$PORT`;
+  GPU base image if the model needs CUDA.
+
+## Step 3 -- (optional) bake in drift monitoring
+If `--drift` (or the user wants it): run the `/openrunner:drift-install` flow on
+the freshly scaffolded `/predict` -- add the non-blocking reporter + a
+`record_inference(...)` after the prediction (latency, confidence, input length).
+
+## Step 4 -- (optional) container + deploy path
+- If `--registry NAME/ns`: show the governed push --
+  `docker build -t registry.openrun.gladia.io/<ns>/<name>:v1 . && openrunner
+  registry login && docker push ...` (the OpenRunner broker proxies to the org's
+  upstream; see `/openrunner:registry`).
+- Note the deploy path: `openrunner gpu run --image <that image> ...` today, and
+  the governed **Deployments** UI (image + GPU + min/max nodes) when available.
+
+## Step 5 -- verify it runs
+- `uvicorn app.main:app` locally (or `python -m app`), `curl /health` -> ok,
+  `curl -X POST /predict` with a sample payload -> a real response.
+- Report the run command, the curl example, and the build/push/deploy one-liners.
+
+## Guardrails
+- Load the model once (startup), not per request; run heavy inference off the
+  event loop.
+- Type the API (pydantic) and add `/health` so it's deploy/LB-ready.
+- Read config (model ref, keys, port) from env -- never hardcode secrets.
+- Keep it minimal + runnable; match the repo's language/stack if one exists.
+"""
+
+
 def install_claude_code() -> list[str]:
     """Install commands in ~/.claude/commands/openrunner/."""
     cmd_dir = Path.home() / ".claude" / "commands" / "openrunner"
@@ -2630,6 +2880,9 @@ def install_claude_code() -> list[str]:
         "derive-dataset.md": DERIVE_DATASET_CMD,
         "launch-gpu.md": LAUNCH_GPU_CMD,
         "registry.md": REGISTRY_CMD,
+        "drift.md": DRIFT_CMD,
+        "drift-install.md": DRIFT_INSTALL_CMD,
+        "build-api.md": BUILD_API_CMD,
     }
 
     files = []
@@ -2832,6 +3085,9 @@ def _install_all_commands_to_dir(cmd_dir: Path, prefix: str = "openrunner:") ->
         "derive-dataset.md": DERIVE_DATASET_CMD,
         "launch-gpu.md": LAUNCH_GPU_CMD,
         "registry.md": REGISTRY_CMD,
+        "drift.md": DRIFT_CMD,
+        "drift-install.md": DRIFT_INSTALL_CMD,
+        "build-api.md": BUILD_API_CMD,
     }
 
     files = []

{openrunner_sdk-2.37.4 → openrunner_sdk-2.37.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "openrunner-sdk"
-version = "2.37.4"
+version = "2.37.6"
 description = "OpenRunner SDK - W&B-compatible ML experiment tracking client"
 readme = "README.md"
 license = {text = "MIT"}