robostats 0.1.0__py3-none-any.whl

robostats/__init__.py

@@ -0,0 +1,115 @@
+"""Statistics and uncertainty quantification for robot policy evaluation.
+
+robostats consumes episode records, one row per rollout, and emits statistics:
+confidence intervals, paired comparisons, and formatted reports. It computes and
+reports; it never judges whether a number is trustworthy.
+
+The path through the package
+----------------------------
+Load records with :func:`load_jsonl` or :func:`load_csv`, naming every column
+explicitly and passing the protocol the run was collected under. Join two runs
+on ``scenario_id`` with :func:`pair`. Compare them with :func:`compare`, which
+returns the estimate, its interval and the p-value together. Render the result
+with :func:`report`.
+
+A caller who already holds records in memory, from a DataFrame or anywhere else,
+builds a :class:`RecordSet` directly and skips the loaders.
+"""
+
+from robostats import adapters
+from robostats.compare import (
+    AUTO_MIN_SHARED,
+    CombinedResult,
+    ComparisonResult,
+    McNemarResult,
+    SensitivityResult,
+    UnpairedResult,
+    combined_difference,
+    compare,
+    compare_combined,
+    compare_unpaired,
+    mcnemar,
+    paired_difference,
+    select_mode,
+    unpaired_difference,
+)
+from robostats.errors import (
+    PresetMismatchError,
+    PresetNotFoundError,
+    RobostatsError,
+    ScenarioSpecMismatchError,
+)
+from robostats.intervals import (
+    ConfidenceInterval,
+    agresti_coull,
+    clopper_pearson,
+    wilson,
+)
+from robostats.io import load_csv, load_jsonl, load_manifest
+from robostats.overlap import OverlapResult, overlap, subset_counts
+from robostats.presets import Preset, describe_preset, preset_names, register_preset
+from robostats.recording import EpisodeRecorder
+from robostats.records import (
+    SCHEMA_VERSION,
+    Alignment,
+    EpisodeRecord,
+    LoadProvenance,
+    PairedResult,
+    Protocol,
+    RecordSet,
+    align,
+    pair,
+)
+from robostats.report import report
+
+#: Package version. A test asserts this equals the version in pyproject.toml,
+#: because the two drift otherwise.
+__version__ = "0.1.0"
+
+__all__ = [
+    "AUTO_MIN_SHARED",
+    "SCHEMA_VERSION",
+    "Alignment",
+    "CombinedResult",
+    "ComparisonResult",
+    "ConfidenceInterval",
+    "EpisodeRecord",
+    "EpisodeRecorder",
+    "LoadProvenance",
+    "McNemarResult",
+    "OverlapResult",
+    "PairedResult",
+    "Preset",
+    "PresetMismatchError",
+    "PresetNotFoundError",
+    "Protocol",
+    "RecordSet",
+    "RobostatsError",
+    "ScenarioSpecMismatchError",
+    "SensitivityResult",
+    "UnpairedResult",
+    "__version__",
+    "adapters",
+    "agresti_coull",
+    "align",
+    "clopper_pearson",
+    "combined_difference",
+    "compare",
+    "compare_combined",
+    "compare_unpaired",
+    "describe_preset",
+    "load_csv",
+    "load_jsonl",
+    "load_manifest",
+    "mcnemar",
+    "overlap",
+    "pair",
+    "paired_difference",
+    "preset_names",
+    "register_preset",
+    "report",
+    "select_mode",
+    "subset_counts",
+    "unpaired_difference",
+    "wilson",
+]

robostats/adapters/__init__.py

@@ -0,0 +1,13 @@
+"""Readers for evaluation harnesses that already persist per-episode outcomes.
+
+Nothing here imports a harness, not even for a type hint. An adapter parses a
+file the harness wrote; it does not talk to the harness, load a policy, or step
+an environment. A harness that discards per-episode outcomes before writing
+cannot have an adapter at all, because there is nothing left in the file to read.
+"""
+
+from __future__ import annotations
+
+__all__ = ["robodojo", "robotwin"]
+
+from robostats.adapters import robodojo, robotwin

robostats/adapters/robodojo.py

@@ -0,0 +1,109 @@
+"""Read a RoboDojo evaluation manifest.
+
+RoboDojo writes one JSON document holding both levels: run settings once at the
+top, and per-episode outcomes in a ``details`` map keyed by index. Those keys are
+positional, so they are ignored; ``layout_id`` is the scenario identity.
+
+This module imports nothing from RoboDojo. It parses a file that RoboDojo already
+wrote, using the published mapping in :data:`robostats.presets.ROBODOJO`.
+
+The episodes the run left out
+-----------------------------
+The manifest records ``completed_layout_ids``, ``abandoned_layout_ids``,
+``unstable_nums`` and ``restart_count``. A success rate computed from ``details``
+has a denominator that excludes the abandoned episodes, which is defensible but
+not what a reader assumes, and is a bias if abandonment correlates with
+difficulty. The counts are carried onto the returned set and stated by
+:func:`~robostats.report.report`. Nothing here adjusts a statistic for them, and
+nothing here judges whether they matter.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from robostats.errors import LoadError
+from robostats.io import load_manifest
+from robostats.presets import ROBODOJO
+from robostats.records import LoadProvenance, Protocol, RecordSet
+
+__all__ = ["load"]
+
+
+def load(path: str | Path, *, protocol: Protocol | None = None) -> RecordSet:
+    """Load a RoboDojo manifest into records.
+
+    Parameters
+    ----------
+    path : str or pathlib.Path
+        The manifest to read.
+    protocol : Protocol or None
+        The protocol this run was collected under. ``None`` means nothing was
+        declared, which is what the manifest itself supports: the protocol is
+        never inferred from a file's contents, so a caller who knows the
+        execution horizon should pass it rather than leave it blank.
+
+    Returns
+    -------
+    RecordSet
+        One record per episode, with ``scenario_id`` composed as
+        ``config_name/layout_id``, ``score`` carried in ``success_detail``
+        unthresholded, and the counts of episodes the run left out recorded on
+        the set.
+
+    Raises
+    ------
+    LoadError
+        If the file is missing, is not a JSON object, or has no ``details``.
+    """
+    location = Path(path)
+    records = load_manifest(location, protocol=protocol or Protocol(), benchmark=ROBODOJO.name)
+    return RecordSet(
+        records.records,
+        scenario_spec=records.scenario_spec,
+        provenance=LoadProvenance(
+            preset=ROBODOJO.name,
+            preset_version=ROBODOJO.version,
+            excluded=_excluded_counts(location),
+        ),
+    )
+
+
+def _excluded_counts(path: Path) -> dict[str, int]:
+    """Count the episodes the run did not put in ``details``.
+
+    Read from the manifest's own bookkeeping, under the names RoboDojo uses. The
+    keys are passed through rather than translated, because renaming another
+    tool's vocabulary makes its numbers harder to check against its own output.
+    """
+    document = _document(path)
+    counts: dict[str, int] = {}
+    for name, key in (
+        ("completed", "completed_layout_ids"),
+        ("abandoned", "abandoned_layout_ids"),
+    ):
+        value = document.get(key)
+        if isinstance(value, list):
+            counts[name] = len(value)
+    for name, key in (("unstable", "unstable_nums"), ("restarts", "restart_count")):
+        value = document.get(key)
+        if isinstance(value, int) and not isinstance(value, bool):
+            counts[name] = value
+    return counts
+
+
+def _document(path: Path) -> dict[str, Any]:
+    """Parse the manifest, or raise a LoadError naming the file."""
+    try:
+        text = path.read_text()
+    except OSError as error:
+        raise LoadError(f"{path}: cannot be read ({error.strerror})") from error
+    try:
+        parsed = json.loads(text)
+    except json.JSONDecodeError as error:
+        raise LoadError(f"{path}: not valid JSON ({error.msg})") from error
+    if not isinstance(parsed, dict):
+        raise LoadError(f"{path}: expected a JSON object, got {type(parsed).__name__}")
+    return parsed

robostats/adapters/robotwin.py

@@ -0,0 +1,78 @@
+"""Record RoboTwin episodes from its trial-end hook.
+
+RoboTwin computes a success rate and writes no per-episode file, so there is
+nothing to read back and no reader here. What it does have is a named interface:
+after every episode it calls ``notify_trial_end(model_client, task_name, seed,
+success)``, which sends ``{task_name, seed, success}`` to the policy server over
+the ``ws`` protocol. Users already write a policy adapter, so handling that
+message is a few lines in a file they own, with no patch to RoboTwin.
+
+This module turns one such payload into a record. It imports nothing from
+RoboTwin and never talks to it: RoboTwin calls the user's handler, and the
+handler calls this.
+
+``task_config`` selects ``demo_clean`` or ``demo_randomized`` and is not in the
+payload, because it is a directory name rather than episode data. It is pinned by
+the caller and composed into ``scenario_id`` ahead of the seed, using the
+composition the ``robotwin`` preset declares, so that recording and loading
+cannot drift apart. Seed 17 under one configuration is a different scene from
+seed 17 under the other; a key that omits the configuration joins the two
+silently.
+
+The hook fires only on the ``ws`` path. A run using the local policy path never
+reaches it, and needs the generic
+:class:`~robostats.recording.EpisodeRecorder` instead. See ``docs/recording.md``.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from robostats.presets import ROBOTWIN
+from robostats.recording import EpisodeRecorder
+from robostats.records import EpisodeRecord
+
+__all__ = ["record_trial_end"]
+
+
+def record_trial_end(
+    recorder: EpisodeRecorder, payload: Mapping[str, Any], *, task_config: str
+) -> EpisodeRecord:
+    """Write one record from a RoboTwin ``trial_end`` payload.
+
+    Parameters
+    ----------
+    recorder : EpisodeRecorder
+        The open recorder to write to. It carries the policy id, the run id and
+        the protocol, none of which are in the payload.
+    payload : Mapping[str, Any]
+        The hook's message, carrying ``task_name``, ``seed`` and ``success``.
+    task_config : str
+        The RoboTwin configuration this run used, for example ``"demo_clean"``.
+        Pinned by the caller because it is a directory name, and composed into
+        ``scenario_id`` ahead of the seed.
+
+    Returns
+    -------
+    EpisodeRecord
+        The record as written.
+
+    Raises
+    ------
+    PresetMismatchError
+        If the payload does not carry the three keys the hook sends. A changed
+        payload is worth stopping for: silently recording two of three fields
+        would produce a file that loads cleanly and means something else.
+    SchemaError
+        If a value violates the record schema, ``success`` not being a ``bool``
+        above all. This aborts the eval run, which is the point: see
+        ``docs/recording.md``.
+    """
+    ROBOTWIN.check_shape(list(payload), "<trial_end payload>")
+    return recorder.record(
+        task_id=str(payload["task_name"]),
+        scenario_id=ROBOTWIN.compose_scenario_id((task_config,), payload),
+        success=payload["success"],
+        seed=payload["seed"] if isinstance(payload["seed"], int) else None,
+    )