onejudge 0.3.2__py3-none-any.whl

onejudge-0.3.2.dist-info/METADATA

@@ -0,0 +1,72 @@
+Metadata-Version: 2.4
+Name: onejudge
+Version: 0.3.2
+Summary: Typed async Python SDK for onejudge
+Keywords: agent,evaluation,judge,harness
+Author: Nick DeRobertis
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Typing :: Typed
+Requires-Dist: jsonschema>=4.18,<5
+Requires-Dist: onejudge-cli==0.3.2
+Requires-Python: >=3.9
+Project-URL: Homepage, https://github.com/nickderobertis/onejudge
+Project-URL: Repository, https://github.com/nickderobertis/onejudge
+Description-Content-Type: text/markdown
+
+# onejudge
+
+Typed async Python access to the `onejudge` CLI. The distribution is
+`onejudge`, the import is `onejudge_sdk`, and each release depends on the
+exact same `onejudge-cli` version.
+
+```console
+pip install onejudge
+```
+
+```python
+import asyncio
+
+from onejudge_sdk import OneJudge
+
+
+async def main() -> None:
+    result = await OneJudge().run(
+        {"provider": {"kind": "oneharness"}},
+        "Review this repository",
+        cwd="/path/to/repository",
+        timeout=3600,
+    )
+    print(result.completed, result.assistant_turns, result.verdicts)
+
+
+asyncio.run(main())
+```
+
+`run` validates the config before starting the CLI, writes a temporary effective
+JSON config (JSON is valid YAML), and always sends the task over stdin with
+`--task -`. It accepts a `provider` override, subprocess `cwd`, additional `env`
+(including `ONEHARNESS_HISTORY_LABELS` and `ONEHARNESS_TIMEOUT`), and a timeout.
+Executable resolution is the constructor's `executable`, then `ONEJUDGE_BIN`,
+then `onejudge` on `PATH`.
+
+Exit 0 and 1 return `RunResult`: `exit_code` and `stderr` remain available, and
+`raw`, `completed`, `verdicts`, `usage`, `assistant_turns`, and `agent_turns`
+cover ai-orchestrator's dispatch needs. Exit 2 (bad config or provider/runtime
+failure) and unexpected nonzero exits raise `OneJudgeProcessError` without
+discarding the exit code or stderr. A caller timeout raises
+`OneJudgeTimeoutError`.
+
+There is no `run_stream` method. `onejudge run` currently emits one final JSON
+report; the JSONL interface in `docs/protocol.md` is the internal provider
+protocol, not a CLI result stream.

onejudge-0.3.2.dist-info/RECORD

@@ -0,0 +1,14 @@
+onejudge_sdk/__init__.py,sha256=misPPdJcb4w_wQFYq1u9bpMsH_0IGT-N_rP_ziHGAE4,449
+onejudge_sdk/_client.py,sha256=jkfmZLybBRuAMZnU4qGOdXn0NQlDZXExFw1fynP4eYg,6194
+onejudge_sdk/_errors.py,sha256=8ZyD2mk8iqnmIdebtHwbnHuJtMvVRd_V4StBbYNLklU,816
+onejudge_sdk/_generated/__init__.py,sha256=kval8EHvlbftbLuIpP9C1qXAAAh0-AlYEb7QyZsvUFk,40
+onejudge_sdk/_generated/input-keys.json,sha256=JlyMaGP_jJVjdP_g2F3AoaenIhQduH__2h6ZaNYB2E8,493
+onejudge_sdk/_generated/schemas.json,sha256=VrtLAQWDRtdcYSD7CwH8yH-R6toOEu9RLedVAItUmNg,20829
+onejudge_sdk/_generated_types.py,sha256=nrzxh4NcXFOTLWbLokxlVI6NGtJg-urGs0BDBoHeZu4,392
+onejudge_sdk/_result.py,sha256=Rl_TexDnH6VV8s70krn-l4Ll3TRex7UItarvH-eOrgo,1261
+onejudge_sdk/_version.py,sha256=2dFqS9d_6JCrfcNn-3y8oLeLOYQJZnizedpcyihEP6A,98
+onejudge_sdk/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+onejudge-0.3.2.dist-info/licenses/LICENSE,sha256=-nAcsuNjoW1EEe6T3WqyGyg4vFrdsiTjA_oo_El8epY,1072
+onejudge-0.3.2.dist-info/WHEEL,sha256=YR4QUxGCbsyoOnWBVTv-p1I4yc3seKGPev46mvmc8Cg,81
+onejudge-0.3.2.dist-info/METADATA,sha256=_9HC1Aj7GyLzbYnWJtyalwgidJvD-HLMlKBwW7g6IPc,2592
+onejudge-0.3.2.dist-info/RECORD,,

onejudge-0.3.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.11.29
+Root-Is-Purelib: true
+Tag: py3-none-any

onejudge-0.3.2.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Nick DeRobertis
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

onejudge_sdk/__init__.py

@@ -0,0 +1,18 @@
+"""Async Python SDK for the onejudge CLI."""
+
+from ._client import OneJudge
+from ._errors import ContractError, OneJudgeProcessError, OneJudgeTimeoutError
+from ._generated_types import RunConfig, RunReport
+from ._result import RunResult
+from ._version import __version__
+
+__all__ = [
+    "ContractError",
+    "OneJudge",
+    "OneJudgeProcessError",
+    "OneJudgeTimeoutError",
+    "RunConfig",
+    "RunReport",
+    "RunResult",
+    "__version__",
+]

onejudge_sdk/_client.py

@@ -0,0 +1,156 @@
+"""Async subprocess client for onejudge's validated JSON interface."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import shutil
+import tempfile
+from collections.abc import Mapping, Sequence
+from functools import cache
+from pathlib import Path
+from typing import Any, Optional, cast
+
+from jsonschema import Draft202012Validator
+from jsonschema.protocols import Validator
+
+from ._errors import ContractError, OneJudgeProcessError, OneJudgeTimeoutError
+from ._generated_types import RunConfig, RunReport
+from ._result import RunResult
+
+_STREAM_LIMIT = 16 * 1024 * 1024
+
+
+def _load_json(name: str) -> dict[str, Any]:
+    path = Path(__file__).with_name("_generated") / name
+    return cast("dict[str, Any]", json.loads(path.read_text(encoding="utf-8")))
+
+
+_SCHEMAS = _load_json("schemas.json")
+_INPUT_KEYS = cast("dict[str, dict[str, str]]", _load_json("input-keys.json"))
+
+
+@cache
+def _validator(root: str) -> Validator:
+    schema = _SCHEMAS[root]
+    Draft202012Validator.check_schema(schema)
+    return Draft202012Validator(schema)
+
+
+def _validate(root: str, value: Any, label: str) -> Any:
+    errors = sorted(_validator(root).iter_errors(value), key=lambda error: list(error.path))
+    if not errors:
+        return value
+    details = []
+    for error in errors:
+        path = ".".join(str(part) for part in error.absolute_path) or "<root>"
+        details.append(f"{path}: {error.message}")
+    raise ContractError(f"{label}: {'; '.join(details)}")
+
+
+def _input(value: Any) -> dict[str, Any]:
+    checked = cast("Mapping[str, Any]", _validate("run_config", value, "invalid onejudge config"))
+    keys = _INPUT_KEYS["run_config"]
+    return {keys.get(key, key): item for key, item in checked.items()}
+
+
+async def _terminate(process: asyncio.subprocess.Process) -> None:
+    if process.returncode is not None:
+        return
+    try:
+        process.terminate()
+    except ProcessLookupError:  # pragma: no cover - OS race after returncode check
+        return
+    try:
+        await asyncio.wait_for(process.wait(), timeout=2)
+    except asyncio.TimeoutError:  # pragma: no cover - defensive hard-kill fallback
+        process.kill()
+        await process.wait()
+
+
+class OneJudge:
+    """Validated async access to an installed onejudge CLI."""
+
+    def __init__(
+        self,
+        *,
+        executable: Optional[str] = None,
+        executable_args: Sequence[str] = (),
+        env: Optional[Mapping[str, str]] = None,
+    ) -> None:
+        self._executable = executable
+        self._executable_args = tuple(executable_args)
+        self._env = dict(env or {})
+
+    def _command(self, args: Sequence[str], path: Optional[str] = None) -> tuple[str, ...]:
+        command = self._executable or os.environ.get("ONEJUDGE_BIN")
+        if command is None:
+            command = shutil.which("onejudge", path=path) or "onejudge"
+        return (command, *self._executable_args, *args)
+
+    async def run(
+        self,
+        config: RunConfig,
+        task: str,
+        *,
+        provider: Optional[str] = None,
+        cwd: Optional[str] = None,
+        env: Optional[Mapping[str, str]] = None,
+        timeout: Optional[float] = None,
+    ) -> RunResult:
+        """Run one task and return exit-faithful process and report data."""
+        parsed = _input(config)
+        if not isinstance(task, str):
+            raise ContractError("invalid onejudge task: expected a string")
+        if provider not in (None, "oneharness", "command", "split"):
+            raise ContractError("invalid onejudge provider: expected oneharness, command, or split")
+        if timeout is not None and (isinstance(timeout, bool) or timeout <= 0):
+            raise ContractError("invalid onejudge timeout: expected a positive number")
+        process_env = {**os.environ, **self._env, **dict(env or {})}
+        for key, item in process_env.items():
+            if not isinstance(key, str) or not key or "=" in key or "\0" in key:
+                raise ContractError(f"invalid environment variable name: {key!r}")
+            if not isinstance(item, str) or "\0" in item:
+                raise ContractError(f"invalid environment variable {key!r}: expected a string")
+        with tempfile.TemporaryDirectory(prefix="onejudge-python-") as directory:
+            config_path = Path(directory) / "effective.onejudge.json"
+            config_path.write_text(json.dumps(parsed), encoding="utf-8")
+            args = ["run", str(config_path), "--task", "-", "--format", "json"]
+            if provider is not None:
+                args.extend(("--provider", provider))
+            process = await asyncio.create_subprocess_exec(
+                *self._command(args, process_env.get("PATH")),
+                cwd=cwd,
+                env=process_env,
+                stdin=asyncio.subprocess.PIPE,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                limit=_STREAM_LIMIT,
+            )
+            try:
+                communication = process.communicate(task.encode())
+                if timeout is None:
+                    stdout_bytes, stderr_bytes = await communication
+                else:
+                    stdout_bytes, stderr_bytes = await asyncio.wait_for(
+                        communication, timeout=timeout
+                    )
+            except asyncio.TimeoutError as error:
+                await _terminate(process)
+                raise OneJudgeTimeoutError(timeout or 0) from error
+            except BaseException:
+                await _terminate(process)
+                raise
+        stderr = stderr_bytes.decode("utf-8", errors="replace")
+        returncode = process.returncode or 0
+        if returncode not in (0, 1):
+            raise OneJudgeProcessError(returncode, stderr)
+        try:
+            value = json.loads(stdout_bytes)
+        except (UnicodeDecodeError, json.JSONDecodeError) as error:
+            raise ContractError(f"onejudge returned invalid JSON: {error}") from error
+        report = cast(
+            "RunReport", _validate("report", value, "invalid onejudge report contract")
+        )
+        return RunResult(exit_code=returncode, stderr=stderr, raw=report)

onejudge_sdk/_errors.py

@@ -0,0 +1,26 @@
+"""Typed public errors raised by the Python SDK."""
+
+from __future__ import annotations
+
+from typing import Optional
+
+
+class ContractError(ValueError):
+    """A value did not match its Rust-owned SDK contract."""
+
+
+class OneJudgeProcessError(RuntimeError):
+    """The onejudge subprocess could not produce a report."""
+
+    def __init__(self, returncode: int, stderr: str) -> None:
+        self.returncode = returncode
+        self.stderr = stderr
+        super().__init__(f"onejudge exited {returncode}: {stderr.strip()}")
+
+
+class OneJudgeTimeoutError(OneJudgeProcessError):
+    """The onejudge subprocess exceeded the caller's timeout."""
+
+    def __init__(self, timeout: float, stderr: str = "") -> None:
+        self.timeout = timeout
+        super().__init__(-1, stderr or f"timed out after {timeout} seconds")

onejudge_sdk/_generated/__init__.py

@@ -0,0 +1 @@
+"""Rust-generated Python SDK assets."""

onejudge_sdk/_generated/input-keys.json

@@ -0,0 +1,22 @@
+{
+  "run_config": {
+    "assessment": "assessment",
+    "bin": "bin",
+    "command": "command",
+    "criterion": "criterion",
+    "done_when": "done_when",
+    "evals": "evals",
+    "judge": "judge",
+    "judge_config": "judge_config",
+    "kind": "kind",
+    "max_turns": "max_turns",
+    "persona": "persona",
+    "provider": "provider",
+    "scale": "scale",
+    "session": "session",
+    "skill": "skill",
+    "system_prompt": "system_prompt",
+    "task": "task",
+    "user": "user"
+  }
+}

onejudge_sdk/_generated/schemas.json

@@ -0,0 +1,588 @@
+{
+  "report": {
+    "$defs": {
+      "JudgeKind": {
+        "description": "The kind of judgement requested.",
+        "oneOf": [
+          {
+            "const": "boolean",
+            "description": "A yes/no verdict.",
+            "type": "string"
+          },
+          {
+            "const": "numeric",
+            "description": "A score on a `[min, max]` scale.",
+            "type": "string"
+          }
+        ]
+      },
+      "JudgeValue": {
+        "anyOf": [
+          {
+            "description": "A boolean verdict.",
+            "type": "boolean"
+          },
+          {
+            "description": "A numeric score.",
+            "format": "double",
+            "type": "number"
+          }
+        ],
+        "description": "The raw value a judge returns: a boolean or a number, matching the query kind.\nDeserialized untagged from the provider's `value` field."
+      },
+      "JudgeVerdict": {
+        "description": "A judge verdict: the raw value (bool or number) plus the stated reason. Part\nof onejudge's versioned [`Report`](crate::Report) contract, so it round-trips\nthrough serde.",
+        "properties": {
+          "reason": {
+            "default": "",
+            "description": "The judge's one-sentence justification.",
+            "type": "string"
+          },
+          "usage": {
+            "anyOf": [
+              {
+                "$ref": "#/$defs/Usage"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "description": "Cost/token usage for the judge call, if reported."
+          },
+          "value": {
+            "$ref": "#/$defs/JudgeValue",
+            "description": "The parsed verdict value."
+          }
+        },
+        "required": [
+          "value",
+          "reason"
+        ],
+        "type": "object"
+      },
+      "Message": {
+        "description": "A single turn in the conversation.",
+        "properties": {
+          "content": {
+            "description": "The turn's text.",
+            "type": "string"
+          },
+          "events": {
+            "description": "The normalized tool events the skill took producing this turn (assistant\nturns only, and only when the harness exposed a tool transcript). Empty\notherwise. Surfaced for post-hoc analysis, streamed live, and rendered\ninto the transcript the judge sees.",
+            "items": {
+              "$ref": "#/$defs/ToolEvent"
+            },
+            "type": "array"
+          },
+          "role": {
+            "$ref": "#/$defs/Role",
+            "description": "Who produced the turn."
+          }
+        },
+        "required": [
+          "role",
+          "content"
+        ],
+        "type": "object"
+      },
+      "NamedVerdict": {
+        "description": "A judge verdict paired with the criterion it scored and the kind of\njudgement, so a serialized report is self-describing.",
+        "properties": {
+          "criterion": {
+            "description": "The plain-English criterion that was scored.",
+            "type": "string"
+          },
+          "kind": {
+            "$ref": "#/$defs/JudgeKind",
+            "description": "Whether it was a boolean or numeric judgement."
+          },
+          "verdict": {
+            "$ref": "#/$defs/JudgeVerdict",
+            "description": "The verdict itself (value, reason, and per-call usage)."
+          }
+        },
+        "required": [
+          "criterion",
+          "kind",
+          "verdict"
+        ],
+        "type": "object"
+      },
+      "Role": {
+        "description": "Who produced a message.",
+        "oneOf": [
+          {
+            "const": "user",
+            "description": "The (real or simulated) user driving the skill.",
+            "type": "string"
+          },
+          {
+            "const": "assistant",
+            "description": "The skill / assistant under test.",
+            "type": "string"
+          },
+          {
+            "const": "system",
+            "description": "System-level framing, if a provider surfaces it.",
+            "type": "string"
+          }
+        ]
+      },
+      "ToolEvent": {
+        "description": "One normalized tool-call / action event the skill took during a turn, lifted\nfrom `oneharness`'s `events` array (its `--events` output). Harness-agnostic,\nso a consumer can inspect shell commands, file edits, and tool uses across any\nharness, not just the final text. `input` is the structured, tool-shaped args\nso a consumer can match on the command string or file path without re-parsing.\n\n`input` is free-form JSON, so `Message`/`Transcript` are `PartialEq` but not\n`Eq`.",
+        "properties": {
+          "index": {
+            "default": 0,
+            "description": "Position within the run, so ordering (\"did X before Y\") is expressible.",
+            "format": "uint",
+            "minimum": 0,
+            "type": "integer"
+          },
+          "input": {
+            "description": "Structured tool arguments (the command, the file path); `None` when none."
+          },
+          "kind": {
+            "description": "`tool_call` (the skill invoked a tool) or `tool_result` (the observation).",
+            "type": "string"
+          },
+          "name": {
+            "description": "Normalized tool name where knowable (e.g. `bash`, `edit_file`); `None` for\na `tool_result` or when the harness did not name it.",
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "output": {
+            "description": "The result/observation text, when the transcript exposed it.",
+            "type": [
+              "string",
+              "null"
+            ]
+          }
+        },
+        "required": [
+          "kind",
+          "index"
+        ],
+        "type": "object"
+      },
+      "Transcript": {
+        "description": "An ordered list of messages.",
+        "properties": {
+          "messages": {
+            "description": "The turns in order, oldest first.",
+            "items": {
+              "$ref": "#/$defs/Message"
+            },
+            "type": "array"
+          }
+        },
+        "required": [
+          "messages"
+        ],
+        "type": "object"
+      },
+      "Usage": {
+        "description": "Token / cost usage for one provider call, or a running total.\n\nEach field is independently optional because not every harness reports every\nsignal (cost is commonly absent on subscription auth, and cache counts only\nwhen the harness surfaces prompt-cache reads/writes). `None` means \"no\nsignal\", never \"zero\" \u2014 so a total stays `None` until something reports a real\nnumber, then accumulates.",
+        "properties": {
+          "cache_read_tokens": {
+            "description": "Prompt tokens served from the provider's prompt cache \u2014 a cheap read of a\npreviously-written prefix \u2014 when the harness reports them. `None` when the\nharness does not surface cache counts, never `0` as a guess.",
+            "format": "uint64",
+            "minimum": 0,
+            "type": [
+              "integer",
+              "null"
+            ]
+          },
+          "cache_write_tokens": {
+            "description": "Prompt tokens written to the provider's prompt cache (a.k.a. cache\ncreation), when the harness reports them. `None` when not surfaced.",
+            "format": "uint64",
+            "minimum": 0,
+            "type": [
+              "integer",
+              "null"
+            ]
+          },
+          "cost_usd": {
+            "description": "Total cost in USD, when reported.",
+            "format": "double",
+            "type": [
+              "number",
+              "null"
+            ]
+          },
+          "input_tokens": {
+            "description": "Prompt/input tokens billed, when reported.",
+            "format": "uint64",
+            "minimum": 0,
+            "type": [
+              "integer",
+              "null"
+            ]
+          },
+          "output_tokens": {
+            "description": "Completion/output tokens billed, when reported.",
+            "format": "uint64",
+            "minimum": 0,
+            "type": [
+              "integer",
+              "null"
+            ]
+          }
+        },
+        "type": "object"
+      }
+    },
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "description": "A judged run: the transcript, the verdicts scored against it, aggregated\nusage, and whether a streaming sink short-circuited the run \u2014 stamped with the\n[`SCHEMA_VERSION`] of the contract that produced it.\n\nBuild one from an [`Outcome`](crate::Outcome) with\n[`Outcome::into_report`](crate::Outcome::into_report), or directly with\n[`Report::new`].",
+    "properties": {
+      "assessment": {
+        "description": "A free-text judgement requested by the caller, if any.",
+        "type": [
+          "string",
+          "null"
+        ]
+      },
+      "completion_reason": {
+        "description": "Why the per-turn supervisor declared the task complete, if it did.",
+        "type": [
+          "string",
+          "null"
+        ]
+      },
+      "schema_version": {
+        "description": "The contract version this report was serialized under.",
+        "format": "uint32",
+        "minimum": 0,
+        "type": "integer"
+      },
+      "stopped_early": {
+        "default": false,
+        "description": "Whether a streaming sink asked to short-circuit the run.",
+        "type": "boolean"
+      },
+      "transcript": {
+        "$ref": "#/$defs/Transcript",
+        "description": "The full conversation transcript, with tool events on assistant turns."
+      },
+      "usage": {
+        "anyOf": [
+          {
+            "$ref": "#/$defs/Usage"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "description": "Aggregated usage across every provider call (`None` if nothing reported)."
+      },
+      "verdicts": {
+        "description": "The verdicts scored against the transcript, in the order they were added.",
+        "items": {
+          "$ref": "#/$defs/NamedVerdict"
+        },
+        "type": "array"
+      }
+    },
+    "required": [
+      "schema_version",
+      "transcript",
+      "stopped_early"
+    ],
+    "title": "Report",
+    "type": "object"
+  },
+  "run_config": {
+    "$defs": {
+      "EvalConfig": {
+        "additionalProperties": false,
+        "description": "One eval scored against the finished transcript.",
+        "properties": {
+          "criterion": {
+            "description": "The plain-English criterion.",
+            "type": "string"
+          },
+          "kind": {
+            "$ref": "#/$defs/JudgeKind",
+            "default": "boolean",
+            "description": "Boolean (a pass/fail verdict) or numeric (a score on a scale)."
+          },
+          "scale": {
+            "default": null,
+            "description": "The inclusive `[min, max]` scale for a numeric eval; ignored (and rejected)\nfor a boolean one. Defaults to `[0, 10]`.",
+            "items": {
+              "format": "double",
+              "type": "number"
+            },
+            "maxItems": 2,
+            "minItems": 2,
+            "type": [
+              "array",
+              "null"
+            ]
+          }
+        },
+        "required": [
+          "criterion"
+        ],
+        "type": "object"
+      },
+      "JudgeKind": {
+        "description": "The kind of judgement requested.",
+        "oneOf": [
+          {
+            "const": "boolean",
+            "description": "A yes/no verdict.",
+            "type": "string"
+          },
+          {
+            "const": "numeric",
+            "description": "A score on a `[min, max]` scale.",
+            "type": "string"
+          }
+        ]
+      },
+      "ProviderConfig": {
+        "additionalProperties": false,
+        "description": "Which backend runs the harness. A flat, strict struct (rather than an\ninternally-tagged enum, which serde cannot pair with `deny_unknown_fields`):\n[`ProviderConfig::resolve`] checks that only the chosen `kind`'s fields are set\nand turns it into a validated [`ProviderSpec`], so a misplaced-but-spelled-right\nkey (e.g. `bin` under `kind: command`) is still a loud error.",
+        "properties": {
+          "bin": {
+            "default": null,
+            "description": "`oneharness`: the `oneharness` binary (default `oneharness`).",
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "command": {
+            "default": null,
+            "description": "`command`: the provider argv (program + args).",
+            "items": {
+              "type": "string"
+            },
+            "type": [
+              "array",
+              "null"
+            ]
+          },
+          "judge": {
+            "anyOf": [
+              {
+                "$ref": "#/$defs/ProviderConfig"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "description": "`split`: the backend that judges and plays the simulated user."
+          },
+          "judge_config": {
+            "default": null,
+            "description": "`oneharness`: the oneharness config file the judge / simulated user run\nunder, passed as `oneharness run --config <path>` (default\n`oneharness.judge.toml`). This is where the judge-side harness/model\nselection lives.",
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "kind": {
+            "$ref": "#/$defs/ProviderKind",
+            "description": "`oneharness` (default) | `command` | `split`."
+          },
+          "skill": {
+            "anyOf": [
+              {
+                "$ref": "#/$defs/ProviderConfig"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "description": "`split`: the backend that runs the agent's turns."
+          }
+        },
+        "type": "object"
+      },
+      "ProviderKind": {
+        "description": "The provider backends the CLI can build. One enum is the single source for\nboth the YAML `kind:` (via `Deserialize`) and the `--provider` flag (via\nclap's `ValueEnum`), so the two surfaces cannot drift.",
+        "oneOf": [
+          {
+            "const": "oneharness",
+            "description": "Shell out to the `oneharness` CLI (the default).",
+            "type": "string"
+          },
+          {
+            "const": "command",
+            "description": "A custom command speaking the JSON-lines protocol.",
+            "type": "string"
+          },
+          {
+            "const": "split",
+            "description": "Compose a skill-runner with a separate judge / simulated-user backend.",
+            "type": "string"
+          }
+        ]
+      },
+      "UserConfig": {
+        "additionalProperties": false,
+        "description": "The simulated user that supervises the agent and drives the loop.",
+        "properties": {
+          "done_when": {
+            "default": null,
+            "description": "A plain-English completion condition; when the judge decides it holds, the\nloop ends. Without it the loop ends at `max_turns` or when the agent\ndeclares itself done.",
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "max_turns": {
+            "default": null,
+            "description": "The assistant-turn cap for this run.",
+            "format": "uint32",
+            "minimum": 0,
+            "type": [
+              "integer",
+              "null"
+            ]
+          },
+          "persona": {
+            "default": "",
+            "description": "How the simulated user behaves (their instructions).",
+            "type": "string"
+          }
+        },
+        "type": "object"
+      }
+    },
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "additionalProperties": false,
+    "description": "The whole YAML config for one run. Field defaults let a minimal file (just a\n`task`) work, while `deny_unknown_fields` makes a typo'd key a hard error\ninstead of a silently-ignored setting.",
+    "properties": {
+      "assessment": {
+        "default": null,
+        "description": "Optional prompt for a free-text judgement of the finished transcript.",
+        "type": [
+          "string",
+          "null"
+        ]
+      },
+      "evals": {
+        "description": "Optional criteria to score the finished transcript with.",
+        "items": {
+          "$ref": "#/$defs/EvalConfig"
+        },
+        "type": "array"
+      },
+      "provider": {
+        "$ref": "#/$defs/ProviderConfig",
+        "description": "Which backend runs the harness (and judges / plays the user).\n\nHarness/model **selection** is no longer a onejudge concern: the agent side\nuses oneharness's discovered `oneharness.toml`, and the judge side uses the\n`provider.judge_config` file (default `oneharness.judge.toml`). Scaffold both\nwith `onejudge init`."
+      },
+      "session": {
+        "default": null,
+        "description": "The caller-owned session name threaded across turns; defaults to\n`onejudge`.",
+        "type": [
+          "string",
+          "null"
+        ]
+      },
+      "skill": {
+        "default": null,
+        "description": "Path to a **skill** directory (containing a `SKILL.md`) whose instruction\nbody seeds the system prompt. Resolved relative to the config file's\ndirectory (or the working dir for a flag-only run). Optional \u2014 combine it\nwith `system_prompt`, use either alone, or neither.",
+        "type": [
+          "string",
+          "null"
+        ]
+      },
+      "system_prompt": {
+        "default": null,
+        "description": "Extra system-prompt text for the harness. When a `skill` is also set, this\ncomes **first** and the skill's body is appended after it.",
+        "type": [
+          "string",
+          "null"
+        ]
+      },
+      "task": {
+        "default": null,
+        "description": "The task to drive to completion. May instead be supplied by `--task`\n(`-` reads stdin); required by the time the plan is built.",
+        "type": [
+          "string",
+          "null"
+        ]
+      },
+      "user": {
+        "anyOf": [
+          {
+            "$ref": "#/$defs/UserConfig"
+          },
+          {
+            "type": "null"
+          }
+        ],
+        "description": "The simulated user / supervisor that drives the loop. Omit for a\nsingle-turn run (the agent answers once)."
+      }
+    },
+    "title": "Config",
+    "type": "object"
+  },
+  "stream_event": {
+    "$defs": {
+      "ToolEvent": {
+        "description": "One normalized tool-call / action event the skill took during a turn, lifted\nfrom `oneharness`'s `events` array (its `--events` output). Harness-agnostic,\nso a consumer can inspect shell commands, file edits, and tool uses across any\nharness, not just the final text. `input` is the structured, tool-shaped args\nso a consumer can match on the command string or file path without re-parsing.\n\n`input` is free-form JSON, so `Message`/`Transcript` are `PartialEq` but not\n`Eq`.",
+        "properties": {
+          "index": {
+            "default": 0,
+            "description": "Position within the run, so ordering (\"did X before Y\") is expressible.",
+            "format": "uint",
+            "minimum": 0,
+            "type": "integer"
+          },
+          "input": {
+            "description": "Structured tool arguments (the command, the file path); `None` when none."
+          },
+          "kind": {
+            "description": "`tool_call` (the skill invoked a tool) or `tool_result` (the observation).",
+            "type": "string"
+          },
+          "name": {
+            "description": "Normalized tool name where knowable (e.g. `bash`, `edit_file`); `None` for\na `tool_result` or when the harness did not name it.",
+            "type": [
+              "string",
+              "null"
+            ]
+          },
+          "output": {
+            "description": "The result/observation text, when the transcript exposed it.",
+            "type": [
+              "string",
+              "null"
+            ]
+          }
+        },
+        "required": [
+          "kind",
+          "index"
+        ],
+        "type": "object"
+      }
+    },
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "description": "One streamed tool event delivered live to an [`Engine::run_streaming`] sink,\ntagged with the turn it belongs to.",
+    "properties": {
+      "event": {
+        "$ref": "#/$defs/ToolEvent",
+        "description": "The normalized tool event."
+      },
+      "turn": {
+        "description": "1-based assistant-turn index within this run.",
+        "format": "uint",
+        "minimum": 0,
+        "type": "integer"
+      }
+    },
+    "required": [
+      "turn",
+      "event"
+    ],
+    "title": "StreamEvent",
+    "type": "object"
+  }
+}

onejudge_sdk/_generated_types.py

@@ -0,0 +1,20 @@
+"""Generated from onejudge. Do not edit."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any, TypedDict
+
+
+class RunConfig(TypedDict, total=False):
+    assessment: Any
+    evals: Sequence[dict[str, Any]]
+    provider: dict[str, Any]
+    session: Any
+    skill: Any
+    system_prompt: Any
+    task: Any
+    user: Any
+
+
+RunReport = dict[str, Any]

onejudge_sdk/_result.py

@@ -0,0 +1,43 @@
+"""Convenient typed view over a validated onejudge report."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from ._generated_types import RunReport
+
+
+@dataclass(frozen=True)
+class RunResult:
+    """A completed or incomplete run, retaining process and raw report data."""
+
+    exit_code: int
+    stderr: str
+    raw: RunReport
+
+    @property
+    def completed(self) -> bool:
+        """Whether onejudge exited with its completed/evals-passed status."""
+        return self.exit_code == 0
+
+    @property
+    def verdicts(self) -> list[dict[str, Any]]:
+        """Return the report's ordered verdicts."""
+        return self.raw.get("verdicts", [])
+
+    @property
+    def usage(self) -> dict[str, Any]:
+        """Return aggregate usage, or an empty mapping when unavailable."""
+        return self.raw.get("usage") or {}
+
+    @property
+    def assistant_turns(self) -> int:
+        """Count assistant turns in the transcript."""
+        messages = self.raw["transcript"]["messages"]
+        return sum(message["role"] == "assistant" for message in messages)
+
+    @property
+    def agent_turns(self) -> int:
+        """Alias assistant turns using ai-orchestrator terminology."""
+        return self.assistant_turns

onejudge_sdk/_version.py

@@ -0,0 +1,3 @@
+"""Package version, stamped from Cargo.toml when packaged. Do not edit."""
+
+__version__ = "0.3.2"

onejudge_sdk/py.typed

@@ -0,0 +1 @@
+