inspect-robots 0.3.1__py3-none-any.whl

inspect_robots/__init__.py

@@ -0,0 +1,108 @@
+"""Inspect Robots — the Inspect AI for robotics.
+
+An evaluation framework for VLA (vision-language-action) models. A benchmark is
+defined once as a [`Task`][inspect_robots.task.Task] and run against any compatible
+pairing of a [`Policy`][inspect_robots.policy.Policy] (the VLA) and an
+[`Embodiment`][inspect_robots.embodiment.Embodiment] (a real robot or simulator).
+
+The public API is everything exported here via ``__all__``. Anything not listed
+(or prefixed with ``_``) is private and carries no stability guarantee.
+"""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("inspect_robots")
+except PackageNotFoundError:  # pragma: no cover - only hit in a non-installed tree
+    __version__ = "0.0.0+unknown"
+
+from inspect_robots.embodiment import (
+    Embodiment,
+    EmbodimentBase,
+    EmbodimentInfo,
+)
+from inspect_robots.eval import eval, eval_set
+from inspect_robots.log import (
+    EvalLog,
+    EvalResults,
+    EvalSpec,
+    EvalStats,
+    SceneResult,
+    read_eval_log,
+)
+from inspect_robots.policy import Policy, PolicyBase, PolicyConfig, PolicyInfo
+from inspect_robots.registry import embodiment, policy, registered, resolve, scorer, sink, task
+from inspect_robots.rollout import TrialRecord
+from inspect_robots.scene import Scene, Target
+from inspect_robots.scorer import (
+    Score,
+    Scorer,
+    episode_length,
+    min_distance_to_goal,
+    operator_scorer,
+    reached_goal_state,
+    success_at_end,
+)
+from inspect_robots.spaces import (
+    ActionSemantics,
+    Box,
+    CameraSpec,
+    ObservationSpace,
+    StateField,
+    StateSpec,
+)
+from inspect_robots.task import Epochs, Task
+from inspect_robots.types import Action, ActionChunk, Observation, StepResult
+
+# The public, stability-guaranteed API. Anything not listed here (or prefixed
+# with ``_``) is private. Authoring a benchmark, policy, or embodiment should only
+# need these names.
+__all__ = [
+    "Action",
+    "ActionChunk",
+    "ActionSemantics",
+    "Box",
+    "CameraSpec",
+    "Embodiment",
+    "EmbodimentBase",
+    "EmbodimentInfo",
+    "Epochs",
+    "EvalLog",
+    "EvalResults",
+    "EvalSpec",
+    "EvalStats",
+    "Observation",
+    "ObservationSpace",
+    "Policy",
+    "PolicyBase",
+    "PolicyConfig",
+    "PolicyInfo",
+    "Scene",
+    "SceneResult",
+    "Score",
+    "Scorer",
+    "StateField",
+    "StateSpec",
+    "StepResult",
+    "Target",
+    "Task",
+    "TrialRecord",
+    "__version__",
+    "embodiment",
+    "episode_length",
+    "eval",
+    "eval_set",
+    "min_distance_to_goal",
+    "operator_scorer",
+    "policy",
+    "reached_goal_state",
+    "read_eval_log",
+    "registered",
+    "resolve",
+    "scorer",
+    "sink",
+    "success_at_end",
+    "task",
+]

inspect_robots/_builtins.py

@@ -0,0 +1,53 @@
+"""Register the in-tree builtin components with the registry.
+
+Imported lazily by [`inspect_robots.registry`][inspect_robots.registry] on first
+lookup, so importing ``inspect_robots`` stays cheap and free of import cycles.
+"""
+
+from __future__ import annotations
+
+from inspect_robots.logging import JsonLogSink, RerunSink
+from inspect_robots.mock import CubePickEmbodiment, NoopPolicy, RandomPolicy, ScriptedPolicy
+from inspect_robots.registry import embodiment, policy, scorer, sink, task
+from inspect_robots.scene import Scene
+from inspect_robots.scorer import (
+    episode_length,
+    min_distance_to_goal,
+    operator_scorer,
+    reached_goal_state,
+    success_at_end,
+)
+from inspect_robots.task import Task
+
+# Embodiments
+embodiment("cubepick")(CubePickEmbodiment)
+
+# Policies
+policy("scripted")(ScriptedPolicy)
+policy("random")(RandomPolicy)
+policy("noop")(NoopPolicy)
+
+# Scorers
+scorer("success_at_end")(success_at_end)
+scorer("episode_length")(episode_length)
+scorer("min_distance_to_goal")(min_distance_to_goal)
+scorer("reached_goal_state")(reached_goal_state)
+scorer("operator")(operator_scorer)
+
+# Sinks
+sink("json")(JsonLogSink)
+sink("rerun")(RerunSink)
+
+
+@task("cubepick-reach")
+def _cubepick_reach(num_scenes: int = 4, max_steps: int = 80) -> Task:
+    """A simple reach benchmark over a handful of seeded cube layouts."""
+    return Task(
+        name="cubepick-reach",
+        scenes=[
+            Scene(id=f"scene-{i}", instruction="reach the cube", init_seed=i)
+            for i in range(num_scenes)
+        ],
+        scorer=success_at_end(),
+        max_steps=max_steps,
+    )

inspect_robots/_defaults.py

@@ -0,0 +1,153 @@
+"""User-level default components for the zero-config CLI (plan 0005).
+
+``inspect-robots "place the spoon on the plate"`` needs a policy and an
+embodiment without flags. They come from, in order: explicit CLI flags
+(handled in ``cli.py``), the ``INSPECT_ROBOTS_POLICY`` /
+``INSPECT_ROBOTS_EMBODIMENT`` environment variables, then the user config
+file ``<config-home>/inspect-robots/config.ini`` (INI via stdlib
+``configparser`` — the core supports py3.10, which has no ``tomllib``).
+
+There is deliberately **no project-local config file**: a checked-in
+``./inspect-robots.ini`` choosing which policy runs on the user's hardware
+would be a trust footgun for a tool that moves physical robots.
+"""
+
+from __future__ import annotations
+
+import configparser
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+ENV_POLICY = "INSPECT_ROBOTS_POLICY"
+ENV_EMBODIMENT = "INSPECT_ROBOTS_EMBODIMENT"
+ENV_SIM_EMBODIMENT = "INSPECT_ROBOTS_SIM_EMBODIMENT"
+
+# Fallbacks for ad-hoc (instruction) runs when neither flag nor config decides.
+ADHOC_SCORER_FALLBACK = "operator"
+ADHOC_MAX_STEPS_FALLBACK = 300
+
+
+def parse_value(text: str) -> Any:
+    """Best-effort scalar parse for ``k=v`` args (bool/int/float/None/str)."""
+    low = text.lower()
+    if low in ("true", "false"):
+        return low == "true"
+    if low in ("none", "null"):
+        return None
+    for caster in (int, float):
+        try:
+            return caster(text)
+        except ValueError:
+            continue
+    return text
+
+
+@dataclass(frozen=True)
+class Defaults:
+    """Resolved user defaults, each with a human-readable source for the run header."""
+
+    policy: str | None = None
+    policy_source: str | None = None
+    embodiment: str | None = None
+    embodiment_source: str | None = None
+    # The sim counterpart --sim swaps in; real hardware is just the default
+    # `embodiment`. Args are kept separate: real-rig args (ports, camera
+    # serials) are wrong for a sim and vice versa.
+    sim_embodiment: str | None = None
+    sim_embodiment_source: str | None = None
+    scorer: str | None = None
+    max_steps: int | None = None
+    policy_args: dict[str, Any] = field(default_factory=dict)
+    embodiment_args: dict[str, Any] = field(default_factory=dict)
+    sim_embodiment_args: dict[str, Any] = field(default_factory=dict)
+
+
+def _config_path(env: Mapping[str, str]) -> Path | None:
+    """The user config file location, derived from ``env`` only (testable)."""
+    if xdg := env.get("XDG_CONFIG_HOME"):
+        home = Path(xdg)
+    elif user_home := env.get("HOME"):
+        home = Path(user_home) / ".config"
+    else:
+        return None
+    return home / "inspect-robots" / "config.ini"
+
+
+def _die(path: Path, problem: str) -> SystemExit:
+    return SystemExit(f"error in {path}: {problem}")
+
+
+def _parse_args_section(parser: configparser.ConfigParser, section: str) -> dict[str, Any]:
+    """An ``[<kind>.args]`` section as parsed kwargs, with ``~`` paths expanded."""
+    if not parser.has_section(section):
+        return {}
+    out: dict[str, Any] = {}
+    for key, raw in parser.items(section):
+        value = parse_value(raw)
+        if isinstance(value, str) and value.startswith("~"):
+            # Checkpoint paths are the flagship use; a literal "~/..." string
+            # would fail silently deep inside a plugin.
+            value = os.path.expanduser(value)
+        out[key] = value
+    return out
+
+
+def _read_config(path: Path) -> Defaults:
+    parser = configparser.ConfigParser(inline_comment_prefixes=(";", "#"))
+    try:
+        with path.open(encoding="utf-8") as fh:
+            parser.read_file(fh)
+    except (configparser.Error, UnicodeDecodeError) as exc:
+        raise _die(path, f"malformed config: {exc}") from exc
+
+    source = str(path)
+    max_steps: int | None = None
+    if raw_steps := parser.get("defaults", "max_steps", fallback=None):
+        parsed = parse_value(raw_steps)
+        if not isinstance(parsed, int) or isinstance(parsed, bool) or parsed < 1:
+            raise _die(path, f"[defaults] max_steps must be an integer >= 1, got {raw_steps!r}")
+        max_steps = parsed
+
+    policy = parser.get("defaults", "policy", fallback=None)
+    embodiment = parser.get("defaults", "embodiment", fallback=None)
+    sim_embodiment = parser.get("defaults", "sim_embodiment", fallback=None)
+    return Defaults(
+        policy=policy,
+        policy_source=source if policy else None,
+        embodiment=embodiment,
+        embodiment_source=source if embodiment else None,
+        sim_embodiment=sim_embodiment,
+        sim_embodiment_source=source if sim_embodiment else None,
+        scorer=parser.get("defaults", "scorer", fallback=None),
+        max_steps=max_steps,
+        policy_args=_parse_args_section(parser, "policy.args"),
+        embodiment_args=_parse_args_section(parser, "embodiment.args"),
+        sim_embodiment_args=_parse_args_section(parser, "sim_embodiment.args"),
+    )
+
+
+def load_defaults(env: Mapping[str, str]) -> Defaults:
+    """Load user defaults: environment variables override the config file.
+
+    ``env`` is injected (pass ``os.environ``) so tests never touch the real
+    home directory. A missing config file yields empty defaults; a malformed
+    or type-invalid one raises ``SystemExit`` naming the file — the
+    zero-config path must never print a traceback at the user.
+    """
+    from dataclasses import replace
+
+    path = _config_path(env)
+    defaults = _read_config(path) if path is not None and path.is_file() else Defaults()
+
+    if policy := env.get(ENV_POLICY):
+        defaults = replace(defaults, policy=policy, policy_source=f"${ENV_POLICY}")
+    if embodiment := env.get(ENV_EMBODIMENT):
+        defaults = replace(defaults, embodiment=embodiment, embodiment_source=f"${ENV_EMBODIMENT}")
+    if sim := env.get(ENV_SIM_EMBODIMENT):
+        defaults = replace(
+            defaults, sim_embodiment=sim, sim_embodiment_source=f"${ENV_SIM_EMBODIMENT}"
+        )
+    return defaults

inspect_robots/approver.py

@@ -0,0 +1,70 @@
+"""The Approver — a safety gate between policy output and the embodiment.
+
+Every action passes through ``Approver.review`` before ``embodiment.step``. This
+is the robotics analog of Inspect AI's ``ApprovalPolicy`` and is more
+safety-critical: an approver may pass, clamp, or veto an action (a veto raises
+[`SafetyAbort`][inspect_robots.errors.SafetyAbort]). In the tracer slice the default approver
+passes everything through; clamping/operator approval land in rollout hardening.
+"""
+
+from __future__ import annotations
+
+from dataclasses import replace
+from typing import Any, Protocol, runtime_checkable
+
+import numpy as np
+
+from inspect_robots.errors import SafetyAbort
+from inspect_robots.spaces import Box
+from inspect_robots.types import Action
+
+
+@runtime_checkable
+class Approver(Protocol):
+    """Reviews an action before it reaches the embodiment.
+
+    May return the action unchanged, return a modified (e.g. clamped) action, or
+    raise [`SafetyAbort`][inspect_robots.errors.SafetyAbort] to halt the eval.
+    """
+
+    def review(self, action: Action, store: dict[str, Any]) -> Action: ...
+
+
+class AutoApprover:
+    """Approve every action unchanged (the permissive default)."""
+
+    def review(self, action: Action, store: dict[str, Any]) -> Action:
+        return action
+
+
+class ClampApprover:
+    """Clamp actions to a box's ``low``/``high`` bounds before they reach hardware.
+
+    One-sided boxes are honored: a ``low``-only box clamps from below, a
+    ``high``-only box from above. A modified action is flagged via
+    ``action.meta["clamped"]`` so the rollout can record an approval event;
+    when nothing clamps, the *same* action object is returned (the rollout
+    detects modification by identity).
+
+    Non-finite values are the safety cases: ``NaN`` anywhere in the action
+    raises [`SafetyAbort`][inspect_robots.errors.SafetyAbort] — a NaN is a
+    poisonous value with no meaningful clamp, and it must never reach hardware.
+    ``±inf`` is *not* an abort: it clamps to the finite bound on that side like
+    any other out-of-range value (and passes through if that side is
+    unbounded).
+    """
+
+    def __init__(self, action_space: Box):
+        self._space = action_space
+
+    def review(self, action: Action, store: dict[str, Any]) -> Action:
+        data = np.asarray(action.data, dtype=np.float64)
+        if bool(np.isnan(data).any()):
+            raise SafetyAbort("ClampApprover: action contains NaN; refusing to pass it on")
+        low, high = self._space.low, self._space.high
+        if low is None and high is None:
+            return action
+        clamped = np.clip(data, low, high)
+        if np.array_equal(clamped, data):
+            return action
+        return replace(action, data=clamped, meta={**dict(action.meta), "clamped": True})