cotterbot 0.1.0__py3-none-any.whl

cotter/__init__.py

@@ -0,0 +1,125 @@
+"""Cotter — compliance testing for AI-controlled robot policies.
+
+Load a trained policy as a black box, run it through standardized test
+categories (performance, safety, regression, adversarial) in a MuJoCo
+simulation, and get structured pass/fail results with statistical
+guarantees.
+"""
+
+from cotter.envs.wrapper import (
+    ACTUATOR_FORCES,
+    CONTACT_COUNT,
+    CONTACT_FORCES,
+    INSTRUMENTED_KEYS,
+    JOINT_VELOCITIES,
+    CotterWrapper,
+)
+from cotter.policy import (
+    Policy,
+    SB3Policy,
+    SpaceMismatchError,
+    TorchPolicy,
+    load_policy,
+    validate_spaces,
+)
+from cotter.report import TestReport
+from cotter.runner import (
+    EpisodeRecord,
+    RolloutSet,
+    make_seed_sequence,
+    rollout_one,
+    run_rollouts,
+    run_rollouts_parallel,
+)
+from cotter.tests.adversarial import (
+    AdversarialResult,
+    NullAdversary,
+    ObservationPerturbationEnv,
+    RandomAdversary,
+    get_adversary,
+    run_adversarial_test,
+    train_adversary,
+)
+from cotter.tests.regression import (
+    RegressionDecision,
+    RegressionResult,
+    mcnemar_exact,
+    wilcoxon_regression,
+)
+from cotter.tests.safety import (
+    SafetyDecision,
+    SafetyLimit,
+    SafetyResult,
+    SafetyViolation,
+    check_step,
+    evaluate_safety,
+)
+from cotter.backends import (
+    BackendFactory,
+    BackendNotAvailableError,
+    GymnasiumBackend,
+    IsaacSimBackend,
+)
+from cotter.stats import clopper_pearson
+from cotter.tests.sprt import SPRT, SPRTDecision, SPRTResult, run_sprt
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # envs
+    "CotterWrapper",
+    "JOINT_VELOCITIES",
+    "ACTUATOR_FORCES",
+    "CONTACT_COUNT",
+    "CONTACT_FORCES",
+    "INSTRUMENTED_KEYS",
+    # policy
+    "Policy",
+    "SB3Policy",
+    "TorchPolicy",
+    "load_policy",
+    "validate_spaces",
+    "SpaceMismatchError",
+    # runner
+    "EpisodeRecord",
+    "RolloutSet",
+    "rollout_one",
+    "run_rollouts",
+    "run_rollouts_parallel",
+    "make_seed_sequence",
+    # performance
+    "SPRT",
+    "SPRTDecision",
+    "SPRTResult",
+    "run_sprt",
+    # stats
+    "clopper_pearson",
+    # backends
+    "BackendFactory",
+    "BackendNotAvailableError",
+    "GymnasiumBackend",
+    "IsaacSimBackend",
+    # safety
+    "SafetyLimit",
+    "SafetyViolation",
+    "SafetyResult",
+    "SafetyDecision",
+    "check_step",
+    "evaluate_safety",
+    # regression
+    "RegressionDecision",
+    "RegressionResult",
+    "mcnemar_exact",
+    "wilcoxon_regression",
+    # adversarial
+    "AdversarialResult",
+    "RandomAdversary",
+    "NullAdversary",
+    "ObservationPerturbationEnv",
+    "run_adversarial_test",
+    "train_adversary",
+    "get_adversary",
+    # report
+    "TestReport",
+    "__version__",
+]

cotter/backends.py

@@ -0,0 +1,125 @@
+"""Simulator backend abstraction.
+
+A backend knows how to turn an env id into a ready-to-test Gymnasium
+environment (instrumented with :class:`~cotter.envs.wrapper.CotterWrapper`
+when possible). This isolates the rest of the framework from *which*
+simulator is behind the Gymnasium API, so the same test battery can run
+against MuJoCo today and other Gymnasium-compatible simulators later.
+
+Backends are looked up by name via :meth:`BackendFactory.from_name`;
+``RunConfig.backend`` selects one (default ``"gymnasium"``). A backend
+that needs an optional dependency raises :class:`BackendNotAvailableError`
+at construction, so an unavailable backend fails immediately with a clear
+message rather than deep inside a run.
+"""
+
+from __future__ import annotations
+
+import abc
+
+import gymnasium as gym
+
+from cotter.envs.registry import make_env_by_id
+from cotter.envs.wrapper import CotterWrapper
+
+
+class BackendNotAvailableError(RuntimeError):
+    """A backend's required simulator/dependency is not importable."""
+
+
+class BackendFactory(abc.ABC):
+    """Interface for simulator backends.
+
+    Concrete backends set a class-level ``backend_name`` (used for
+    registration and lookup) and implement :meth:`make_env`.
+    """
+
+    backend_name: str = ""
+    _registry: dict[str, type["BackendFactory"]] = {}
+
+    def __init_subclass__(cls, **kwargs) -> None:
+        super().__init_subclass__(**kwargs)
+        if cls.backend_name:
+            BackendFactory._registry[cls.backend_name] = cls
+
+    @abc.abstractmethod
+    def make_env(self, env_id: str) -> gym.Env:
+        """Create an instrumented environment for ``env_id``."""
+
+    def name(self) -> str:
+        return self.backend_name
+
+    @classmethod
+    def from_name(cls, name: str) -> "BackendFactory":
+        """Construct the backend registered under ``name``.
+
+        Raises ``ValueError`` for an unknown name and
+        :class:`BackendNotAvailableError` if the named backend's
+        dependencies are missing (surfaced from its constructor).
+        """
+        if name not in cls._registry:
+            raise ValueError(
+                f"unknown backend '{name}' (available: {sorted(cls._registry)})"
+            )
+        return cls._registry[name]()
+
+    @classmethod
+    def available(cls) -> list[str]:
+        return sorted(cls._registry)
+
+
+class GymnasiumBackend(BackendFactory):
+    """Default backend: standard Gymnasium/MuJoCo via ``gym.make``."""
+
+    backend_name = "gymnasium"
+
+    def make_env(self, env_id: str) -> gym.Env:
+        env = make_env_by_id(env_id)
+        try:
+            return CotterWrapper(env)
+        except TypeError:
+            # not MuJoCo-backed: safety instrumentation is unavailable, but
+            # performance/regression/adversarial still work on the raw env
+            return env
+
+
+class IsaacSimBackend(BackendFactory):
+    """NVIDIA Isaac Sim backend (optional, GPU/cluster only).
+
+    Isaac Sim exposes a Gymnasium-compatible API but requires the
+    ``omni.isaac.gym`` package, which is not importable on CPU-only
+    machines. Construction fails fast when it is absent; when present,
+    envs are built through Isaac's Gymnasium interface and instrumented
+    with :class:`CotterWrapper` like any other MuJoCo/Gymnasium env.
+
+    This path is not exercised on the CPU development machine; it is
+    validated on a GPU cluster separately.
+    """
+
+    backend_name = "isaac-sim"
+
+    def __init__(self) -> None:
+        import importlib.util
+
+        # find_spec returns None for a missing leaf, but raises
+        # ModuleNotFoundError when an intermediate package (omni) is absent;
+        # both mean the backend is unavailable.
+        try:
+            spec = importlib.util.find_spec("omni.isaac.gym")
+        except ModuleNotFoundError:
+            spec = None
+        if spec is None:
+            raise BackendNotAvailableError(
+                "the isaac-sim backend requires the 'omni.isaac.gym' package "
+                "from NVIDIA Isaac Sim, which is not installed. Install Isaac "
+                "Sim on a CUDA machine, or use the default 'gymnasium' backend."
+            )
+
+    def make_env(self, env_id: str) -> gym.Env:  # pragma: no cover - no Isaac on CI
+        # Isaac registers Gymnasium-compatible envs; build and instrument
+        # them through the same path as any MuJoCo env.
+        env = make_env_by_id(env_id)
+        try:
+            return CotterWrapper(env)
+        except TypeError:
+            return env

cotter/cli.py

@@ -0,0 +1,221 @@
+"""`cotter` command-line interface.
+
+    cotter run --policy artifacts/victim.zip --config run.yaml
+    cotter compare --baseline a.zip --candidate b.zip --config run.yaml
+
+Exit codes: 0 = all executed categories passed (compare: no regression),
+1 = at least one category failed (compare: regression detected),
+2 = configuration/usage error.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from cotter import __version__
+from cotter.config import ConfigError, RegressionConfig, RunConfig, load_config
+from cotter.pipeline import run_from_config
+from cotter.policy import SpaceMismatchError
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="cotter",
+        description="Compliance testing for AI-controlled robot policies.",
+    )
+    parser.add_argument("--version", action="version", version=f"cotter {__version__}")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    run = subparsers.add_parser(
+        "run", help="run the test categories declared in a YAML config"
+    )
+    run.add_argument(
+        "--policy", type=Path, required=True,
+        help="policy under test (SB3 .zip or torch .pt)",
+    )
+    run.add_argument(
+        "--config", type=Path, required=True,
+        help="YAML config declaring test categories and parameters",
+    )
+    run.add_argument(
+        "--env", default=None,
+        help="Gymnasium env id (overrides the config's 'env')",
+    )
+    run.add_argument(
+        "--report", type=Path, default=None,
+        help="JSON report path (overrides the config's 'report')",
+    )
+    run.add_argument(
+        "--quiet", action="store_true", help="suppress progress output"
+    )
+
+    compare = subparsers.add_parser(
+        "compare",
+        help="run only the regression test between a baseline and a candidate",
+    )
+    compare.add_argument("--baseline", type=Path, required=True, help="baseline policy")
+    compare.add_argument("--candidate", type=Path, required=True, help="candidate policy")
+    compare.add_argument(
+        "--config", type=Path, required=True,
+        help="YAML config supplying env, success, base_seed, and regression params",
+    )
+    compare.add_argument("--env", default=None, help="Gymnasium env id (overrides config)")
+    compare.add_argument("--quiet", action="store_true", help="suppress progress output")
+
+    list_envs = subparsers.add_parser(
+        "list-envs", help="list available Gymnasium env ids grouped by package"
+    )
+    list_envs.add_argument(
+        "--filter", default=None, help="only show env ids containing this substring"
+    )
+
+    zoo = subparsers.add_parser("zoo", help="inspect the cached adversary zoo")
+    zoo.add_argument("--root", type=Path, default=None, help="zoo root (default ~/.cotter/zoo)")
+    zoo_sub = zoo.add_subparsers(dest="zoo_command", required=True)
+    zoo_list = zoo_sub.add_parser("list", help="list cached adversaries")
+    zoo_list.add_argument("--env", default=None, help="filter by env id")
+    zoo_sub.add_parser("prune", help="remove entries whose artifact files are missing")
+    return parser
+
+
+def cmd_run(args: argparse.Namespace) -> int:
+    log = (lambda msg: None) if args.quiet else print
+    try:
+        cfg = load_config(args.config)
+    except (ConfigError, FileNotFoundError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    if args.env is not None:
+        cfg.env = args.env
+    if args.report is not None:
+        cfg.report = args.report
+
+    try:
+        report = run_from_config(args.policy, cfg, log=log)
+    except (FileNotFoundError, SpaceMismatchError, ValueError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+
+    print(report.summary())
+    return 0 if report.overall_passed else 1
+
+
+def cmd_compare(args: argparse.Namespace) -> int:
+    log = (lambda msg: None) if args.quiet else print
+    try:
+        cfg = load_config(args.config)
+    except (ConfigError, FileNotFoundError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+
+    # Regression-only run: keep the loaded regression params (n_pairs,
+    # alpha, n_workers) but override the baseline with the CLI argument and
+    # drop every other category so only the comparison executes.
+    reg = cfg.regression or RegressionConfig()
+    reg.baseline = args.baseline
+    reg_only = RunConfig(
+        env=args.env or cfg.env,
+        success=cfg.success,
+        algo=cfg.algo,
+        base_seed=cfg.base_seed,
+        regression=reg,
+    )
+
+    try:
+        report = run_from_config(args.candidate, reg_only, log=log)
+    except (FileNotFoundError, SpaceMismatchError, ValueError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+
+    print(report.summary())
+    regressed = any(r.passed is False for r in report.results)
+    return 1 if regressed else 0
+
+
+def cmd_list_envs(args: argparse.Namespace) -> int:
+    from collections import defaultdict
+
+    import gymnasium as gym
+
+    from cotter.envs.registry import register_extension_envs
+
+    register_extension_envs()  # make gymnasium-robotics ids visible
+
+    groups: dict[str, set[str]] = defaultdict(set)
+    for env_id, spec in gym.registry.items():
+        if args.filter and args.filter.lower() not in env_id.lower():
+            continue
+        entry = spec.entry_point
+        if isinstance(entry, str):
+            package = entry.split(":", 1)[0].split(".", 1)[0]
+        elif entry is not None:
+            package = getattr(entry, "__module__", "unknown").split(".", 1)[0]
+        else:
+            package = "unknown"
+        groups[package].add(env_id)
+
+    total = sum(len(ids) for ids in groups.values())
+    scope = f" matching '{args.filter}'" if args.filter else ""
+    if total == 0:
+        print(f"no envs{scope}")
+        return 0
+    print(f"{total} env id(s){scope}:")
+    for package in sorted(groups):
+        ids = sorted(groups[package])
+        print(f"\n{package} ({len(ids)}):")
+        for env_id in ids:
+            print(f"  {env_id}")
+    return 0
+
+
+def cmd_zoo(args: argparse.Namespace) -> int:
+    from cotter.zoo import AdversaryZoo
+
+    zoo = AdversaryZoo(args.root) if args.root else AdversaryZoo()
+
+    if args.zoo_command == "list":
+        entries = zoo.entries(args.env)
+        scope = f" for env '{args.env}'" if args.env else ""
+        if not entries:
+            print(f"no cached adversaries{scope} (zoo root: {zoo.root})")
+            return 0
+        print(f"{len(entries)} cached adversar{'y' if len(entries) == 1 else 'ies'}"
+              f"{scope} (zoo root: {zoo.root}):")
+        for e in entries:
+            missing = "" if (zoo.root / e.path).exists() else "  [MISSING ARTIFACT]"
+            print(f"  {e.env_id}  eps={e.epsilon:g}  victim={e.victim_hash}  "
+                  f"{e.algo}  {e.created_at}{missing}")
+            print(f"      {e.path}")
+        return 0
+
+    if args.zoo_command == "prune":
+        removed = zoo.prune()
+        if not removed:
+            print(f"nothing to prune (zoo root: {zoo.root})")
+            return 0
+        print(f"pruned {len(removed)} entr{'y' if len(removed) == 1 else 'ies'} "
+              f"with missing artifacts:")
+        for e in removed:
+            print(f"  {e.env_id}  eps={e.epsilon:g}  victim={e.victim_hash}")
+        return 0
+
+    raise AssertionError(f"unhandled zoo command {args.zoo_command}")  # pragma: no cover
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    if args.command == "run":
+        return cmd_run(args)
+    if args.command == "compare":
+        return cmd_compare(args)
+    if args.command == "zoo":
+        return cmd_zoo(args)
+    if args.command == "list-envs":
+        return cmd_list_envs(args)
+    raise AssertionError(f"unhandled command {args.command}")  # pragma: no cover
+
+
+if __name__ == "__main__":
+    sys.exit(main())

cotter/compliance/__init__.py

@@ -0,0 +1,33 @@
+"""Regulatory compliance document generation — PAID TIER (license required).
+
+This module is a deliberate stub. The open-source Cotter engine produces
+the *evidence* (structured TestReport pass/fail results with statistical
+guarantees); turning that evidence into signed regulatory documentation
+— EU Machinery Regulation 2027 technical files, ISO 10218 conformity
+records — is the licensed commercial layer.
+
+The public import path is stable so downstream code and docs can be
+written against it today::
+
+    from cotter.compliance import EUMachineryReg2027
+
+    generator = EUMachineryReg2027(report)   # raises LicenseRequiredError
+
+Every constructor here raises :class:`LicenseRequiredError` with
+activation instructions. Nothing in this package performs real document
+generation in the open-source distribution.
+"""
+
+from cotter.compliance.base import (
+    ComplianceGenerator,
+    LicenseRequiredError,
+    EUMachineryReg2027,
+    ISO10218,
+)
+
+__all__ = [
+    "ComplianceGenerator",
+    "LicenseRequiredError",
+    "EUMachineryReg2027",
+    "ISO10218",
+]

cotter/compliance/base.py

@@ -0,0 +1,74 @@
+"""Paid-tier compliance generators — stub implementations.
+
+# paid tier — license required
+#
+# These classes define the commercial API surface but perform no real
+# work in the open-source distribution: constructing any generator
+# raises LicenseRequiredError. The shapes here (standard name, version,
+# required category coverage) are intentionally concrete so the boundary
+# between the free engine and the paid layer is legible.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from cotter.report import TestReport
+
+_ACTIVATION_HINT = (
+    "Regulatory document generation is a licensed feature and is not "
+    "included in the open-source Cotter distribution. The open engine "
+    "produces the pass/fail evidence (TestReport); the compliance layer "
+    "renders it into a regulator-ready technical file. Request access at "
+    "https://github.com/yih0nk/cotter (see the commercial tier) or set a "
+    "valid COTTER_LICENSE_KEY."
+)
+
+
+class LicenseRequiredError(RuntimeError):
+    """Raised when a paid-tier compliance feature is used without a license."""
+
+    def __init__(self, standard: str) -> None:
+        super().__init__(
+            f"'{standard}' compliance generation requires a Cotter license. "
+            f"{_ACTIVATION_HINT}"
+        )
+        self.standard = standard
+
+
+class ComplianceGenerator:
+    """Base class for regulatory document generators (paid tier).
+
+    Subclasses declare the standard they cover and the test categories
+    that standard's evidence requires. The open-source stub validates
+    nothing and generates nothing — it refuses at construction.
+    """
+
+    standard: str = "unknown"
+    version: str = "0"
+    required_categories: tuple[str, ...] = ()
+
+    def __init__(self, report: "TestReport") -> None:
+        # The license check happens before any work, so downstream code
+        # gets a clear, immediate signal rather than a late failure.
+        raise LicenseRequiredError(self.standard)
+
+    def generate(self, output_path) -> None:  # pragma: no cover - unreachable in OSS
+        raise LicenseRequiredError(self.standard)
+
+
+class EUMachineryReg2027(ComplianceGenerator):
+    """EU Machinery Regulation (2023/1230) technical file generator (paid)."""
+
+    standard = "EU Machinery Regulation 2027"
+    version = "2023/1230"
+    required_categories = ("performance", "safety", "adversarial")
+
+
+class ISO10218(ComplianceGenerator):
+    """ISO 10218 industrial-robot safety conformity generator (paid)."""
+
+    standard = "ISO 10218"
+    version = "2011"
+    required_categories = ("safety", "performance")