daishi 0.1.0__tar.gz

daishi-0.1.0/.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+dist/
+*.egg-info/

daishi-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Arkeous LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

daishi-0.1.0/PKG-INFO

@@ -0,0 +1,88 @@
+Metadata-Version: 2.5
+Name: daishi
+Version: 0.1.0
+Summary: Typed client for the Daishi Studio API: scenarios, runs, series batches and paired experiments, with retries and waiting helpers.
+Project-URL: Homepage, https://daishi.ai
+Project-URL: Documentation, https://daishi.ai/docs/developer
+Author: Arkeous LLC
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agents,ai,benchmark,daishi,evaluation,llm,multi-agent
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# daishi
+
+Typed client for the Studio API of [Daishi](https://daishi.ai), an independent agent testing and evaluation platform: save scenarios, launch runs, series batches and paired experiments on your own model keys, and read the scored results.
+
+```bash
+pip install daishi
+```
+
+Python 3.10 or later. No dependencies: the standard library only.
+
+## Quick start
+
+In the [Studio](https://daishi.ai/studio), open **Account > Developer**, create an access token with **runs: launch and cancel** ticked, and export it:
+
+```bash
+export DAISHI_TOKEN=dsk_...        # macOS, Linux
+$env:DAISHI_TOKEN = "dsk_..."      # Windows PowerShell
+```
+
+```python
+from daishi import Daishi
+
+daishi = Daishi()  # reads DAISHI_TOKEN
+
+run = daishi.launch({
+    "scenario_id": "daishi:famine-v1",
+    "roster": [{"model": "<provider>/<model>"}, {"model": "<provider>/<model>"}],
+    "max_spend_usd": 5,
+})["run"]
+
+done = daishi.wait_for_run(run["run_id"])
+for agent in (done["results"] or {}).get("agents", []):
+    print(agent["name"], agent.get("fitness_index"), agent.get("grade"))
+```
+
+Seats play on the provider keys stored on your account. Every plan can read and validate; creating, launching and cancelling need a paid plan.
+
+## What it does
+
+- **One method per API operation**, named after its operation id: `get_run`, `list_runs`, `launch`, `cancel`, `get_batch`, `create_scenario`, `create_experiment`, `get_experiment` and the rest. Responses are plain dicts typed as `TypedDict`s generated from the API's OpenAPI document, so editors and type checkers know every field.
+- **Errors** raise `DaishiError` with the API's `code` (`unknown_run`, `plan_quota`, ...), `status`, `message`, `retry_after_seconds`, and `docs`, a link to what the code means.
+- **Retries** are safe by construction. A 429 is retried after the seconds its `Retry-After` header names, and a cancel refused with `run_launching` after a pause, on any method, because the server refused before doing anything. A gateway error or a dropped connection is retried on GET only, so a write is never sent twice.
+- **Waiting.** `wait_for_run`, `wait_for_batch` and `wait_for_experiment` poll every 30 seconds until the run, batch or experiment ends. `wait_for_run` also waits for a finished run's results, which arrive when the match archives. Pass `timeout=` (seconds) for a deadline; it raises `TimeoutError`. To be told instead of asking, set up [run-end notices](https://daishi.ai/docs/developer/runs#run-end-notices).
+
+```python
+from daishi import Daishi, DaishiError
+
+try:
+    daishi.get_run("run_...")
+except DaishiError as err:
+    if err.code != "unknown_run":
+        raise
+    print(err.docs)
+```
+
+## Options
+
+`Daishi(token=None, *, base_url="https://daishi.ai/api/v1", max_retries=3, timeout=60.0)`. `token` defaults to `DAISHI_TOKEN`; `max_retries=0` turns retrying off; `timeout` is seconds per attempt.
+
+## Types
+
+Every schema in the API is importable: `Run`, `RunResults`, `Scenario`, `Skill`, `BatchReport`, `Experiment`, `ExperimentReport`, the request bodies (`RunBody`, `ScenarioBody`, `ExperimentBody`, ...), and `ErrorCode`. Fields whose shape follows the scenario, the provider or the meter are `JsonObject` (`dict[str, Any]`). A few operations (`me`, `usage`, the estimates, logs and invites) answer `JsonObject` until the API types them; `run_log` answers the log's NDJSON text.
+
+## Docs
+
+[Developer docs](https://daishi.ai/docs/developer) and the [API reference](https://daishi.ai/docs/developer/rest).
+
+MIT licensed.

daishi-0.1.0/README.md

@@ -0,0 +1,68 @@
+# daishi
+
+Typed client for the Studio API of [Daishi](https://daishi.ai), an independent agent testing and evaluation platform: save scenarios, launch runs, series batches and paired experiments on your own model keys, and read the scored results.
+
+```bash
+pip install daishi
+```
+
+Python 3.10 or later. No dependencies: the standard library only.
+
+## Quick start
+
+In the [Studio](https://daishi.ai/studio), open **Account > Developer**, create an access token with **runs: launch and cancel** ticked, and export it:
+
+```bash
+export DAISHI_TOKEN=dsk_...        # macOS, Linux
+$env:DAISHI_TOKEN = "dsk_..."      # Windows PowerShell
+```
+
+```python
+from daishi import Daishi
+
+daishi = Daishi()  # reads DAISHI_TOKEN
+
+run = daishi.launch({
+    "scenario_id": "daishi:famine-v1",
+    "roster": [{"model": "<provider>/<model>"}, {"model": "<provider>/<model>"}],
+    "max_spend_usd": 5,
+})["run"]
+
+done = daishi.wait_for_run(run["run_id"])
+for agent in (done["results"] or {}).get("agents", []):
+    print(agent["name"], agent.get("fitness_index"), agent.get("grade"))
+```
+
+Seats play on the provider keys stored on your account. Every plan can read and validate; creating, launching and cancelling need a paid plan.
+
+## What it does
+
+- **One method per API operation**, named after its operation id: `get_run`, `list_runs`, `launch`, `cancel`, `get_batch`, `create_scenario`, `create_experiment`, `get_experiment` and the rest. Responses are plain dicts typed as `TypedDict`s generated from the API's OpenAPI document, so editors and type checkers know every field.
+- **Errors** raise `DaishiError` with the API's `code` (`unknown_run`, `plan_quota`, ...), `status`, `message`, `retry_after_seconds`, and `docs`, a link to what the code means.
+- **Retries** are safe by construction. A 429 is retried after the seconds its `Retry-After` header names, and a cancel refused with `run_launching` after a pause, on any method, because the server refused before doing anything. A gateway error or a dropped connection is retried on GET only, so a write is never sent twice.
+- **Waiting.** `wait_for_run`, `wait_for_batch` and `wait_for_experiment` poll every 30 seconds until the run, batch or experiment ends. `wait_for_run` also waits for a finished run's results, which arrive when the match archives. Pass `timeout=` (seconds) for a deadline; it raises `TimeoutError`. To be told instead of asking, set up [run-end notices](https://daishi.ai/docs/developer/runs#run-end-notices).
+
+```python
+from daishi import Daishi, DaishiError
+
+try:
+    daishi.get_run("run_...")
+except DaishiError as err:
+    if err.code != "unknown_run":
+        raise
+    print(err.docs)
+```
+
+## Options
+
+`Daishi(token=None, *, base_url="https://daishi.ai/api/v1", max_retries=3, timeout=60.0)`. `token` defaults to `DAISHI_TOKEN`; `max_retries=0` turns retrying off; `timeout` is seconds per attempt.
+
+## Types
+
+Every schema in the API is importable: `Run`, `RunResults`, `Scenario`, `Skill`, `BatchReport`, `Experiment`, `ExperimentReport`, the request bodies (`RunBody`, `ScenarioBody`, `ExperimentBody`, ...), and `ErrorCode`. Fields whose shape follows the scenario, the provider or the meter are `JsonObject` (`dict[str, Any]`). A few operations (`me`, `usage`, the estimates, logs and invites) answer `JsonObject` until the API types them; `run_log` answers the log's NDJSON text.
+
+## Docs
+
+[Developer docs](https://daishi.ai/docs/developer) and the [API reference](https://daishi.ai/docs/developer/rest).
+
+MIT licensed.

daishi-0.1.0/pyproject.toml

@@ -0,0 +1,34 @@
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[project]
+name = "daishi"
+version = "0.1.0"
+description = "Typed client for the Daishi Studio API: scenarios, runs, series batches and paired experiments, with retries and waiting helpers."
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Arkeous LLC" }]
+requires-python = ">=3.10"
+dependencies = []
+keywords = ["daishi", "ai", "agents", "evaluation", "benchmark", "llm", "multi-agent"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "Intended Audience :: Science/Research",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+  "Typing :: Typed",
+]
+
+[project.urls]
+Homepage = "https://daishi.ai"
+Documentation = "https://daishi.ai/docs/developer"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/daishi"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src/daishi", "README.md", "LICENSE", "pyproject.toml"]

daishi-0.1.0/src/daishi/__init__.py

@@ -0,0 +1,18 @@
+"""Typed client for the Daishi Studio API.
+
+    from daishi import Daishi
+
+    client = Daishi()  # reads DAISHI_TOKEN
+    run = client.launch({"scenario_id": "daishi:famine-v1", "roster": [{"model": "<provider>/<model>"}]})["run"]
+    done = client.wait_for_run(run["run_id"])
+
+Docs: https://daishi.ai/docs/developer
+"""
+
+from . import _generated
+from ._client import DEFAULT_BASE_URL, VERSION, Daishi, DaishiError
+from ._generated import *  # noqa: F401,F403  (the API's types)
+
+__version__ = VERSION
+
+__all__ = ["Daishi", "DaishiError", "DEFAULT_BASE_URL", "VERSION", *_generated.__all__]

daishi-0.1.0/src/daishi/_client.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+import email.utils
+import json
+import os
+import random
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any, Callable, Dict, Optional, TypeVar
+
+from ._generated import GetBatchResponse, GetExperimentResponse, GetRunResponse, _Operations
+
+__all__ = ["Daishi", "DaishiError", "DEFAULT_BASE_URL", "VERSION"]
+
+VERSION = "0.1.0"
+DEFAULT_BASE_URL = "https://daishi.ai/api/v1"
+
+_RUN_DONE = {"finished", "failed", "cancelled"}
+_EXPERIMENT_DONE = {"finished", "cancelled"}
+_GATEWAY = {502, 503, 504}
+
+T = TypeVar("T")
+
+
+class DaishiError(Exception):
+    """A refusal from the API, or a transport failure after the retries ran out."""
+
+    def __init__(self, status: int, code: str, message: str, retry_after_seconds: Optional[float] = None, body: Any = None) -> None:
+        super().__init__(message)
+        #: HTTP status; 0 when no response arrived.
+        self.status = status
+        #: The API's error code (``unknown_run``, ``rate_limited``, ...), or ``http_<status>`` when the body carried none.
+        self.code = code
+        self.message = message
+        #: Seconds the ``Retry-After`` header asked for, when it sent one.
+        self.retry_after_seconds = retry_after_seconds
+        #: The parsed error body, when there was one.
+        self.body = body
+
+    @property
+    def docs(self) -> str:
+        """Where the code is explained."""
+        return f"https://daishi.ai/docs/developer/errors#{self.code}"
+
+    def __repr__(self) -> str:
+        return f"DaishiError(status={self.status}, code={self.code!r}, message={self.message!r})"
+
+
+class Daishi(_Operations):
+    """The Daishi Studio API.
+
+    Every operation is a method (``get_run``, ``launch``, ``create_scenario``, ...)
+    typed from the API's OpenAPI document; the ``wait_for_*`` helpers poll until
+    a run, batch or experiment ends.
+
+    Retries: a 429 is retried after the seconds its ``Retry-After`` names, and a
+    ``run_launching`` refusal after a short pause, on any method, because the
+    server refused before doing anything. A gateway error (502, 503, 504) or a
+    dropped connection is retried on GET only: a write that may have landed is
+    never sent twice.
+
+    :param token: A Studio access token (``dsk_...``). Default: the ``DAISHI_TOKEN`` environment variable.
+    :param base_url: Default ``https://daishi.ai/api/v1``.
+    :param max_retries: Retries after a refusal that is safe to repeat. Default 3; 0 turns retrying off.
+    :param timeout: Per-attempt timeout in seconds. Default 60.
+    """
+
+    def __init__(self, token: Optional[str] = None, *, base_url: str = DEFAULT_BASE_URL, max_retries: int = 3, timeout: float = 60.0) -> None:
+        self.token = token if token is not None else (os.environ.get("DAISHI_TOKEN") or None)
+        self.base_url = base_url.rstrip("/")
+        self.max_retries = max_retries
+        self.timeout = timeout
+
+    def __repr__(self) -> str:
+        return f"Daishi(base_url={self.base_url!r})"
+
+    def _request(self, method: str, path: str, *, query: Optional[Dict[str, Any]] = None, body: Any = None, text: bool = False) -> Any:
+        url = self.base_url + ("" if path == "/" else path)
+        params = {k: v for k, v in (query or {}).items() if v is not None}
+        if params:
+            url += "?" + urllib.parse.urlencode(params)
+        headers = {
+            "Accept": "application/x-ndjson, application/json" if text else "application/json",
+            "User-Agent": f"daishi-python/{VERSION}",
+        }
+        if self.token:
+            headers["Authorization"] = f"Bearer {self.token}"
+        data = None
+        if body is not None:
+            headers["Content-Type"] = "application/json"
+            data = json.dumps(body).encode("utf-8")
+
+        attempt = 0
+        while True:
+            can_retry = attempt < self.max_retries
+            req = urllib.request.Request(url, data=data, headers=headers, method=method)
+            try:
+                with urllib.request.urlopen(req, timeout=self.timeout) as res:
+                    raw = res.read().decode("utf-8")
+                    return raw if text else json.loads(raw)
+            except urllib.error.HTTPError as err:
+                raw = err.read().decode("utf-8", "replace")
+                try:
+                    parsed = json.loads(raw)
+                except ValueError:
+                    parsed = None
+                parsed_obj = parsed if isinstance(parsed, dict) else {}
+                code = parsed_obj.get("error") or f"http_{err.code}"
+                retry_after = _parse_retry_after(err.headers.get("Retry-After") if err.headers else None)
+                retryable = err.code == 429 or code == "run_launching" or (method == "GET" and err.code in _GATEWAY)
+                if retryable and can_retry:
+                    time.sleep(retry_after if retry_after is not None else _backoff(attempt))
+                    attempt += 1
+                    continue
+                message = parsed_obj.get("message") or raw[:200] or err.reason
+                raise DaishiError(err.code, code, str(message), retry_after, parsed) from None
+            except (urllib.error.URLError, TimeoutError, ConnectionError) as err:
+                if method == "GET" and can_retry:
+                    time.sleep(_backoff(attempt))
+                    attempt += 1
+                    continue
+                reason = getattr(err, "reason", err)
+                raise DaishiError(0, "network_error", f"{method} {urllib.parse.urlsplit(url).path} failed: {reason}") from err
+
+    def wait_for_run(self, id: str, *, interval: float = 30.0, timeout: Optional[float] = None, on_poll: Optional[Callable[[GetRunResponse], None]] = None) -> GetRunResponse:
+        """Poll a run until it ends (finished, failed or cancelled) and return the last read.
+
+        A finished run is read until its ``results`` arrive, which is when the match archives.
+        Raises ``TimeoutError`` if ``timeout`` seconds pass first.
+        """
+        return self._poll(lambda: self.get_run(id), lambda r: r["run"]["status"] in _RUN_DONE and (r["run"]["status"] != "finished" or r["results"] is not None), interval, timeout, on_poll)
+
+    def wait_for_batch(self, id: str, *, interval: float = 30.0, timeout: Optional[float] = None, on_poll: Optional[Callable[[GetBatchResponse], None]] = None) -> GetBatchResponse:
+        """Poll a series batch until no trial is still to play."""
+        return self._poll(lambda: self.get_batch(id), lambda r: r["batch"]["pending"] == 0, interval, timeout, on_poll)
+
+    def wait_for_experiment(self, id: str, *, interval: float = 30.0, timeout: Optional[float] = None, on_poll: Optional[Callable[[GetExperimentResponse], None]] = None) -> GetExperimentResponse:
+        """Poll a paired experiment until it finishes or is cancelled."""
+        return self._poll(lambda: self.get_experiment(id), lambda r: r["experiment"]["status"] in _EXPERIMENT_DONE, interval, timeout, on_poll)
+
+    def _poll(self, read: Callable[[], T], done: Callable[[T], bool], interval: float, timeout: Optional[float], on_poll: Optional[Callable[[T], None]]) -> T:
+        deadline = None if timeout is None else time.monotonic() + timeout
+        while True:
+            answer = read()
+            if on_poll:
+                on_poll(answer)
+            if done(answer):
+                return answer
+            if deadline is not None and time.monotonic() + interval > deadline:
+                raise TimeoutError(f"still not done after {timeout} seconds")
+            time.sleep(interval)
+
+
+def _parse_retry_after(header: Optional[str]) -> Optional[float]:
+    if not header:
+        return None
+    try:
+        seconds = float(header)
+        return seconds if seconds >= 0 else None
+    except ValueError:
+        pass
+    try:
+        at = email.utils.parsedate_to_datetime(header)
+    except (TypeError, ValueError):
+        return None
+    return max(0.0, at.timestamp() - time.time())
+
+
+def _backoff(attempt: int) -> float:
+    """1 s, 2 s, 4 s ... with up to a quarter of jitter, capped at 30 s."""
+    base = min(30.0, 2.0**attempt)
+    return base + random.random() * base * 0.25