physim-gate 0.2.0__py3-none-any.whl

physim_gate/__init__.py

@@ -0,0 +1,62 @@
+"""PhySim Gate: simulator-agnostic regression gates for physical AI.
+
+Typical library use::
+
+    from physim_gate import compare_runs, load_config, load_run
+
+    result = compare_runs(load_run(baseline), load_run(candidate), load_config(gate))
+    print(result.verdict, result.summary)
+"""
+
+from __future__ import annotations
+
+__version__ = "0.2.0"
+
+from physim_gate.comparison import (
+    ComparisonResult,
+    Evidence,
+    ExitCode,
+    MetricComparison,
+    Verdict,
+    load_comparison,
+)
+from physim_gate.config import GateConfig, MetricGate, load_config
+from physim_gate.engine import compare_runs
+from physim_gate.schema import (
+    FAILURE_RATE_METRIC,
+    Direction,
+    Domain,
+    MetricSpec,
+    RunResult,
+    SchemaError,
+    Trial,
+    TrialStatus,
+    dump_json,
+    load_run,
+    write_run,
+)
+
+__all__ = [
+    "FAILURE_RATE_METRIC",
+    "ComparisonResult",
+    "Direction",
+    "Domain",
+    "Evidence",
+    "ExitCode",
+    "GateConfig",
+    "MetricComparison",
+    "MetricGate",
+    "MetricSpec",
+    "RunResult",
+    "SchemaError",
+    "Trial",
+    "TrialStatus",
+    "Verdict",
+    "__version__",
+    "compare_runs",
+    "dump_json",
+    "load_comparison",
+    "load_config",
+    "load_run",
+    "write_run",
+]

physim_gate/__main__.py

@@ -0,0 +1,3 @@
+from physim_gate.cli import main
+
+main()

physim_gate/adapters/__init__.py

@@ -0,0 +1,23 @@
+"""Adapters convert simulator output into PhySim Gate run results."""
+
+from physim_gate.adapters.base import (
+    BUILTIN_ADAPTERS,
+    ENTRY_POINT_GROUP,
+    Adapter,
+    AdapterError,
+    AdapterInfo,
+    discover,
+    get_adapter,
+    parse_options,
+)
+
+__all__ = [
+    "BUILTIN_ADAPTERS",
+    "ENTRY_POINT_GROUP",
+    "Adapter",
+    "AdapterError",
+    "AdapterInfo",
+    "discover",
+    "get_adapter",
+    "parse_options",
+]

physim_gate/adapters/base.py

@@ -0,0 +1,142 @@
+"""The adapter plugin contract and discovery.
+
+An adapter turns a simulator's native output into a :class:`~physim_gate.schema.RunResult`.
+Third-party packages register adapters under the ``physim_gate.adapters`` entry-point group::
+
+    [project.entry-points."physim_gate.adapters"]
+    my-sim = "my_package.adapter:MySimAdapter"
+
+The entry point must resolve to a zero-argument callable (usually a class) that returns an
+object satisfying :class:`Adapter`. No change to PhySim Gate itself is needed.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from importlib import import_module
+from importlib.metadata import EntryPoint, entry_points
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+from physim_gate.schema import RunResult
+
+ENTRY_POINT_GROUP = "physim_gate.adapters"
+
+#: Built-in adapters, available even when package metadata is missing (e.g. a source checkout).
+BUILTIN_ADAPTERS: dict[str, str] = {
+    "json": "physim_gate.adapters.json_adapter:JsonAdapter",
+    "synthetic": "physim_gate.adapters.synthetic:SyntheticAdapter",
+    "gymnasium": "physim_gate.adapters.gymnasium:GymnasiumAdapter",
+    "lerobot": "physim_gate.adapters.lerobot:LeRobotAdapter",
+}
+
+
+class AdapterError(Exception):
+    """An adapter could not be found, loaded, or could not convert its input."""
+
+
+@runtime_checkable
+class Adapter(Protocol):
+    """What every adapter provides.
+
+    ``load`` receives the path given on the command line and any ``--adapter-option key=value``
+    pairs. It must return a validated :class:`RunResult` or raise :class:`AdapterError` (or
+    :class:`~physim_gate.schema.SchemaError`) with a message that tells the user what to fix.
+    """
+
+    name: str
+    description: str
+
+    def load(self, source: Path, options: Mapping[str, str]) -> RunResult: ...
+
+
+@dataclass(frozen=True)
+class AdapterInfo:
+    name: str
+    target: str
+    origin: str
+    error: str | None = None
+
+
+def _entry_points() -> list[EntryPoint]:
+    return list(entry_points(group=ENTRY_POINT_GROUP))
+
+
+def discover() -> dict[str, AdapterInfo]:
+    """All known adapters by name. Conflicting registrations are reported as errors."""
+    found = {
+        name: AdapterInfo(name=name, target=target, origin="built-in")
+        for name, target in BUILTIN_ADAPTERS.items()
+    }
+    for ep in _entry_points():
+        origin = ep.dist.name if ep.dist is not None else "unknown distribution"
+        existing = found.get(ep.name)
+        if existing is None:
+            found[ep.name] = AdapterInfo(name=ep.name, target=ep.value, origin=origin)
+        elif existing.target != ep.value:
+            found[f"{ep.name} ({origin})"] = AdapterInfo(
+                name=ep.name,
+                target=ep.value,
+                origin=origin,
+                error=f"name conflicts with the adapter from {existing.origin}; ignored",
+            )
+    return dict(sorted(found.items()))
+
+
+def _resolve(target: str) -> object:
+    module_name, _, attr = target.partition(":")
+    obj: object = import_module(module_name)
+    for part in attr.split(".") if attr else []:
+        obj = getattr(obj, part)
+    return obj
+
+
+def get_adapter(name: str) -> Adapter:
+    """Instantiate the adapter registered as ``name``."""
+    info = discover().get(name)
+    if info is None or info.error is not None:
+        available = ", ".join(n for n, i in discover().items() if i.error is None)
+        raise AdapterError(f"unknown adapter {name!r}; available: {available}")
+    try:
+        factory = _resolve(info.target)
+    except Exception as exc:
+        raise AdapterError(
+            f"adapter {name!r} ({info.target}, from {info.origin}) failed to import: "
+            f"{type(exc).__name__}: {exc}"
+        ) from exc
+    if not callable(factory):
+        raise AdapterError(f"adapter {name!r} ({info.target}) is not callable")
+    try:
+        adapter = factory()
+    except Exception as exc:
+        raise AdapterError(
+            f"adapter {name!r} ({info.target}) failed to initialize: {type(exc).__name__}: {exc}"
+        ) from exc
+    if not isinstance(adapter, Adapter):
+        raise AdapterError(
+            f"adapter {name!r} ({info.target}) does not implement the Adapter protocol "
+            "(needs 'name', 'description' and 'load(source, options)')"
+        )
+    return adapter
+
+
+def parse_options(pairs: list[str]) -> dict[str, str]:
+    """Parse repeated ``key=value`` strings."""
+    options: dict[str, str] = {}
+    for pair in pairs:
+        key, sep, value = pair.partition("=")
+        if not sep or not key:
+            raise AdapterError(f"adapter option {pair!r} must look like key=value")
+        options[key.strip()] = value.strip()
+    return options
+
+
+def reject_unknown_options(adapter: str, options: Mapping[str, str], known: set[str]) -> None:
+    unknown = sorted(set(options) - known)
+    if unknown:
+        accepted = ", ".join(sorted(known)) or "none"
+        raise AdapterError(
+            f"adapter {adapter!r} does not accept option(s) {', '.join(unknown)}; "
+            f"accepted: {accepted}"
+        )

physim_gate/adapters/gymnasium.py

@@ -0,0 +1,224 @@
+"""Optional Gymnasium integration: roll out a policy over seeds and record a run result.
+
+Requires the ``gymnasium`` extra (``pip install "physim-gate[gymnasium]"``). Only CPU
+environments that need no extra system packages (for example the classic-control suite) are
+tested. Gymnasium's own reproducibility caveats apply: seeding ``reset`` fixes the initial
+state, not every source of non-determinism in every environment.
+
+From Python::
+
+    from physim_gate.adapters.gymnasium import run_gymnasium
+    run = run_gymnasium("CartPole-v1", my_policy, seeds=range(30), run_id="cand-42")
+
+From the CLI, point the ``gymnasium`` adapter at a YAML spec::
+
+    env_id: CartPole-v1
+    policy: my_package.policies:balance   # callable(observation) -> action, or "random"
+    episodes: 30
+"""
+
+from __future__ import annotations
+
+import importlib
+import platform
+from collections.abc import Callable, Iterable, Mapping
+from pathlib import Path
+from typing import Annotated, Any
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+
+from physim_gate.adapters.base import AdapterError, reject_unknown_options
+from physim_gate.provenance import capture_environment, capture_git_source, utc_now
+from physim_gate.schema import (
+    Direction,
+    Domain,
+    Failure,
+    Identifier,
+    InputError,
+    MetricSpec,
+    RunResult,
+    RunStatus,
+    SchemaError,
+    Simulator,
+    Trial,
+    TrialStatus,
+    format_validation_error,
+)
+
+Policy = Callable[[Any], Any]
+
+
+def _require_gymnasium() -> Any:
+    try:
+        return importlib.import_module("gymnasium")
+    except ImportError as exc:
+        raise AdapterError(
+            "the gymnasium adapter needs the optional dependency: "
+            'pip install "physim-gate[gymnasium]"'
+        ) from exc
+
+
+def run_gymnasium(
+    env_id: str,
+    policy: Policy | None,
+    *,
+    seeds: Iterable[int],
+    run_id: str,
+    policy_id: str | None = None,
+    max_steps: int | None = None,
+    env_kwargs: Mapping[str, Any] | None = None,
+    domain: Domain = Domain.GENERIC,
+) -> RunResult:
+    """Roll out ``policy`` once per seed and return a run result.
+
+    ``policy`` maps an observation to an action. ``None`` samples uniformly from the action space,
+    seeded per episode. Exceptions raised by the policy or the environment mark that trial as
+    ``error`` instead of aborting the run.
+    """
+    gym = _require_gymnasium()
+    env = gym.make(env_id, **dict(env_kwargs or {}))
+    trials: list[Trial] = []
+    seed_list = list(seeds)
+    try:
+        for seed in seed_list:
+            trials.append(_episode(env, policy, seed, max_steps))
+    finally:
+        env.close()
+
+    errors = sum(1 for t in trials if t.status is not TrialStatus.OK)
+    return RunResult(
+        run_id=run_id,
+        created_at=utc_now(),
+        domain=domain,
+        simulator=Simulator(name="gymnasium", version=str(gym.__version__)),
+        scenario=Identifier(id=env_id),
+        policy=Identifier(id=policy_id or _policy_name(policy)),
+        seeds=seed_list,
+        source=capture_git_source(),
+        environment=capture_environment(implementation=platform.python_implementation()),
+        status=RunStatus.PARTIAL if errors else RunStatus.COMPLETED,
+        metrics={
+            "episode_return": MetricSpec(
+                direction=Direction.HIGHER_IS_BETTER,
+                description="Sum of rewards over the episode.",
+            ),
+            "episode_length": MetricSpec(
+                direction=Direction.HIGHER_IS_BETTER,
+                unit="steps",
+                description="Steps until termination or truncation.",
+            ),
+            "terminated": MetricSpec(
+                direction=Direction.LOWER_IS_BETTER,
+                description="1 if the episode ended in a terminal state, 0 if truncated.",
+            ),
+        },
+        trials=trials,
+    )
+
+
+def _episode(env: Any, policy: Policy | None, seed: int, max_steps: int | None) -> Trial:
+    total = 0.0
+    steps = 0
+    terminated = truncated = False
+    try:
+        observation, _ = env.reset(seed=seed)
+        env.action_space.seed(seed)
+        while not (terminated or truncated):
+            action = env.action_space.sample() if policy is None else policy(observation)
+            observation, reward, terminated, truncated, _ = env.step(action)
+            total += float(reward)
+            steps += 1
+            if max_steps is not None and steps >= max_steps:
+                truncated = True
+    except Exception as exc:
+        return Trial(
+            trial_id=f"seed-{seed}",
+            seed=seed,
+            status=TrialStatus.ERROR,
+            failure=Failure(kind=type(exc).__name__, message=str(exc)),
+            metrics={"episode_return": None, "episode_length": None, "terminated": None},
+        )
+    return Trial(
+        trial_id=f"seed-{seed}",
+        seed=seed,
+        metrics={
+            "episode_return": total,
+            "episode_length": float(steps),
+            "terminated": 1.0 if terminated else 0.0,
+        },
+    )
+
+
+def _policy_name(policy: Policy | None) -> str:
+    if policy is None:
+        return "random"
+    module = getattr(policy, "__module__", None) or "unknown"
+    name = getattr(policy, "__qualname__", None) or type(policy).__name__
+    return f"{module}:{name}"
+
+
+class GymnasiumSpec(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    env_id: str
+    policy: str = "random"
+    run_id: str
+    policy_id: str | None = None
+    episodes: Annotated[int, Field(ge=1, le=100_000)] = 30
+    first_seed: int = 0
+    max_steps: Annotated[int, Field(ge=1)] | None = None
+    env_kwargs: dict[str, Any] = Field(default_factory=dict)
+    domain: Domain = Domain.GENERIC
+
+
+def load_policy(reference: str) -> Policy | None:
+    """Resolve ``"random"`` or ``"module:attribute"`` to a policy callable."""
+    if reference == "random":
+        return None
+    module_name, sep, attr = reference.partition(":")
+    if not sep or not attr:
+        raise AdapterError(f"policy {reference!r} must be 'random' or 'module:attribute'")
+    try:
+        obj: Any = importlib.import_module(module_name)
+        for part in attr.split("."):
+            obj = getattr(obj, part)
+    except (ImportError, AttributeError) as exc:
+        raise AdapterError(f"cannot load policy {reference!r}: {exc}") from exc
+    if not callable(obj):
+        raise AdapterError(f"policy {reference!r} is not callable")
+    policy: Policy = obj
+    return policy
+
+
+class GymnasiumAdapter:
+    name = "gymnasium"
+    description = "Runs a policy in a Gymnasium environment over seeds (optional extra, CPU)."
+
+    def load(self, source: Path, options: Mapping[str, str]) -> RunResult:
+        reject_unknown_options(self.name, options, {"run_id", "policy", "episodes"})
+        try:
+            data = yaml.safe_load(source.read_text(encoding="utf-8"))
+        except OSError as exc:
+            raise InputError(f"{source}: cannot read file ({exc.strerror or exc})") from exc
+        except yaml.YAMLError as exc:
+            raise SchemaError(f"{source}: invalid YAML: {exc}") from exc
+        if not isinstance(data, dict):
+            raise SchemaError(f"{source}: expected a mapping at the top level")
+        data = {**data, **options}
+        try:
+            spec = GymnasiumSpec.model_validate(data)
+        except ValidationError as exc:
+            raise SchemaError(
+                f"{source}: not a valid gymnasium spec", format_validation_error(exc)
+            ) from exc
+        return run_gymnasium(
+            spec.env_id,
+            load_policy(spec.policy),
+            seeds=range(spec.first_seed, spec.first_seed + spec.episodes),
+            run_id=spec.run_id,
+            policy_id=spec.policy_id or spec.policy,
+            max_steps=spec.max_steps,
+            env_kwargs=spec.env_kwargs,
+            domain=spec.domain,
+        )

physim_gate/adapters/json_adapter.py

@@ -0,0 +1,18 @@
+"""Reads results that are already in the PhySim Gate run schema."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from pathlib import Path
+
+from physim_gate.adapters.base import reject_unknown_options
+from physim_gate.schema import RunResult, load_run
+
+
+class JsonAdapter:
+    name = "json"
+    description = "PhySim Gate run-result JSON (schema 1.x). The default."
+
+    def load(self, source: Path, options: Mapping[str, str]) -> RunResult:
+        reject_unknown_options(self.name, options, set())
+        return load_run(source)