simulo-interfaces 0.22.0__tar.gz → 0.23.0__tar.gz

{simulo_interfaces-0.22.0/src/simulo_interfaces.egg-info → simulo_interfaces-0.23.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: simulo-interfaces
-Version: 0.22.0
+Version: 0.23.0
 Summary: Typed Python contracts for the Simulo cloud robotics SDK and managed simulation runtime.
 Author-email: Simulo Team <team@simulo.ai>
 License: BSD-3-Clause

{simulo_interfaces-0.22.0 → simulo_interfaces-0.23.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "simulo-interfaces"
-version = "0.22.0"
+version = "0.23.0"
 description = "Typed Python contracts for the Simulo cloud robotics SDK and managed simulation runtime."
 readme = "PYPI.md"
 requires-python = ">=3.11"

{simulo_interfaces-0.22.0 → simulo_interfaces-0.23.0}/src/simulo/interfaces/authoring/record.py

@@ -44,10 +44,13 @@ fingerprint that overrides the auto-collected one when present).
 
 from __future__ import annotations
 
+import math
 import os
+import sys
 from dataclasses import dataclass, field
+from numbers import Real
 from pathlib import Path
-from typing import Any, Dict, Literal, Mapping, Optional, Sequence, Tuple, Union
+from typing import Any, Dict, Literal, Mapping, Optional, Sequence, SupportsFloat, Tuple, Union
 
 # All include_* field names this config exposes. Centralizing avoids drift
 # between the dataclass declaration, _PROFILE_DEFAULTS, and resolve_includes.
@@ -253,7 +256,14 @@ class RecordConfig:
     video_fallback: Literal["software", "error"] = "software"
     video_codec: Literal["h264", "hevc", "av1"] = "h264"
     video_bitrate: Optional[str] = "8M"
-    video_fps: Optional[int] = 30
+    # ``None`` leaves cadence to the frame producer. The encoder then uses its
+    # documented 30 fps default, but the recorder omits ``video_fps`` from
+    # /run/start because packet timestamps, not an encoder setting, describe
+    # that producer-owned cadence. Callers such as evaluation filming set the
+    # measured cadence explicitly.
+    # Values must also fit a finite IEEE-754 double because both supported
+    # encoders and the MCAP JSON record use that representation.
+    video_fps: Optional[SupportsFloat] = None
     video_gop: Optional[int] = 30
     video_storage: Literal["embedded", "sidecar"] = "embedded"
     # When True (default), the recorder writes a parallel
@@ -300,6 +310,24 @@ class RecordConfig:
             raise ValueError(f"RecordConfig.profile must be one of {VALID_PROFILES}; got {self.profile!r}")
         if self.record_every_n_steps < 1:
             raise ValueError(f"RecordConfig.record_every_n_steps must be >= 1; got {self.record_every_n_steps}")
+        try:
+            video_fps = None if self.video_fps is None else float(self.video_fps)
+        except (OverflowError, TypeError, ValueError):
+            video_fps = None
+        if self.video_fps is not None and (
+            isinstance(self.video_fps, bool)
+            or not isinstance(self.video_fps, Real)
+            or video_fps is None
+            or not math.isfinite(video_fps)
+            # 0.1 fps is a policy floor that keeps rates sensible; ffmpeg's
+            # rational conversion only rounds to zero far below it.
+            or video_fps < 0.1
+            or video_fps > sys.float_info.max
+        ):
+            raise ValueError(
+                "RecordConfig.video_fps must be a finite real number representable by the encoder and at least 0.1 "
+                f"when set; got {self.video_fps!r}"
+            )
         # env_indices: accept any sequence of ints, including a torch tensor —
         # but normalize to a tuple of ints for hashability + reproducibility.
         try:

{simulo_interfaces-0.22.0 → simulo_interfaces-0.23.0}/src/simulo/interfaces/authoring/robot.py

@@ -7,7 +7,7 @@ and reset paths all reach through a live engine handle), so its
 *implementation* cannot live in a zero-dependency package. What CAN live here
 is its declared public surface: this Protocol names exactly what authoring
 code may rely on when it holds a robot — construction-time configuration,
-joint/body queries, command targets, state reads, and reset.
+joint/body queries, command targets, default state, live state reads, and reset.
 
 The backend's concrete ``Robot`` class explicitly inherits this Protocol
 (nominal conformance). Enforcement is the standard four layers for a mirror —
@@ -23,15 +23,16 @@ The Protocol stays structural for any other implementer.
 
 ``state`` is DECLARED, as ``RobotStateProtocol`` — the grouped live-state
 contract. It is the supported, typed way to read a robot's simulation state
-(``robot.state.joint_positions``, ``robot.state.pose``, …): six read-only,
+(``robot.state.joint_positions``, ``robot.state.pose``, …): eight read-only,
 batched members whose names are plain English rather than engine vocabulary.
-The members are LIVE VIEWS — each read goes straight through to the
-simulation's own buffers, at the same cost as reaching through the
-``internals`` escape hatch, with no copy and no per-access container
-allocation — so reading them inside a training loop is free of hidden
-overhead. Unlike ``internals``, these six names ARE a compatibility promise:
-they keep working across engine upgrades, because the backend maps each one
-to whatever the engine calls it.
+Six members are zero-copy live views of the simulation's buffers. The two
+base-frame velocity members are different: the engine rotates the current
+world-frame velocities into the base frame on every read, returning a new
+tensor. That small computation and allocation are the cost of those two
+accesses, and writing into either result does not reach the simulation.
+Unlike ``internals``, all eight names ARE a compatibility promise: they keep
+working across engine upgrades, because the backend maps each one to whatever
+the engine calls it.
 
 Deliberate scoping notes, so they are not mistaken for drift (each is an
 entry in the backend conformance test's documented-omissions allowlist):
@@ -68,27 +69,31 @@ from simulo.interfaces.runtime.tensors import TensorLike
 
 @runtime_checkable
 class RobotStateProtocol(Protocol):
-    """The live robot-state contract — batched, read-only, zero-copy.
+    """The live robot-state contract — batched and read-only.
 
     Obtained from a robot as ``robot.state``; never constructed directly.
-    Each member is a live view of the simulation's own state buffers: reading
-    one returns the current values without copying, and the same access works
-    identically inside a training loop's hot path. All members carry the
-    environment batch as their leading dimension (the scene's ``num_envs``).
+    Six members (``pose``, the two world-frame velocities, and the three
+    ``joint_*`` members) are zero-copy live views of the simulation's own
+    buffers. ``linear_velocity_in_base_frame`` and
+    ``angular_velocity_in_base_frame`` are computed from the current base
+    orientation on every read. Each of those reads returns a new tensor,
+    costs a small computation, and cannot modify the simulation when written
+    into. All members carry the environment batch as their leading dimension
+    (the scene's ``num_envs``).
 
     **Read a member each time you need it — do not hold the returned tensor
-    across steps, and do not write into it.** The tensor IS the simulation's
-    own memory: the engine refreshes most of these buffers by rebinding them
-    to new tensors, so a tensor held across a step can silently serve stale
-    values, and writing into one writes into the simulation. The accessor
-    (``robot.state`` itself) may be held freely; the tensors its members
-    return should be re-read at each use.
+    across steps, and do not write into it.** The six live-view tensors share
+    simulation memory, and writing into one writes into the simulation. The
+    engine can also rebind those buffers, while the base-frame members are
+    recomputed, so a tensor held across a step can silently serve stale values.
+    The accessor (``robot.state`` itself) may be held freely; the tensors its
+    members return should be re-read at each use.
 
     Members raise ``RuntimeError`` until the simulation runtime has attached
     the robot — read them from ``on_start`` onward. The ``state`` accessor
     itself is always available and is the same object on every access.
 
-    These six names are a compatibility promise: they keep working across
+    These eight names are a compatibility promise: they keep working across
     engine upgrades. For anything not named here, the ``internals`` escape
     hatch remains available — explicitly unstable, at your own risk.
     """
@@ -111,6 +116,24 @@ class RobotStateProtocol(Protocol):
         ``[wx, wy, wz]``."""
         ...
 
+    @property
+    def linear_velocity_in_base_frame(self) -> TensorLike:
+        """Base linear velocity expressed in the base frame, shape
+        ``(num_envs, 3)`` in m/s with columns ``[vx, vy, vz]``. Available
+        from ``on_start`` onward. Computed from the current base orientation
+        on every read; each read returns a new tensor, costs a small
+        computation, and writing into it does not reach the simulation."""
+        ...
+
+    @property
+    def angular_velocity_in_base_frame(self) -> TensorLike:
+        """Base angular velocity expressed in the base frame, shape
+        ``(num_envs, 3)`` in rad/s with columns ``[wx, wy, wz]``. Available
+        from ``on_start`` onward. Computed from the current base orientation
+        on every read; each read returns a new tensor, costs a small
+        computation, and writing into it does not reach the simulation."""
+        ...
+
     @property
     def joint_positions(self) -> TensorLike:
         """Joint positions (rad or m), shape ``(num_envs, num_joints)``."""
@@ -146,7 +169,7 @@ class RobotProtocol(Protocol):
     """Model source (USD or URDF) defining the robot."""
 
     initial_pose: Pose
-    """Pose the robot spawns at, in the world frame (identity when not given)."""
+    """Pose the robot spawns at (identity when not given). For a robot copied into each environment, the position is measured from that environment's origin, and default_pose reports it with the origin added. For some catalog robots, such as the cartpole, the identity pose keeps the catalog model's own spawn height; any other pose replaces it. A robot added under a Transform spawns at the Transform's position."""
 
     collision_enabled: bool
     """Whether collision detection is enabled."""
@@ -203,12 +226,45 @@ class RobotProtocol(Protocol):
         """The grouped live robot state — the supported, typed state surface.
 
         Always available (the accessor is the same object on every access);
-        its members read straight through to the simulation's state buffers
-        and raise ``RuntimeError`` until the simulation runtime has attached
-        the robot — read them from ``on_start`` onward.
+        six members read straight through to the simulation's state buffers,
+        while the two base-frame velocity members are computed into new
+        tensors on every read. Members raise ``RuntimeError`` until the
+        simulation runtime has attached the robot — read them from
+        ``on_start`` onward.
         """
         ...
 
+    @property
+    def default_joint_positions(self) -> TensorLike:
+        """Joint positions that :meth:`reset` writes, shape
+        ``(num_envs, num_joints)`` in rad for hinge joints and m for slider
+        joints. Columns use the same order as ``state.joint_positions`` and
+        the indices returned by :meth:`find_joints`. Available from
+        ``on_start`` onward. Each read returns a new tensor, so writing into
+        it cannot change a later reset."""
+        ...
+
+    @property
+    def default_joint_velocities(self) -> TensorLike:
+        """Joint velocities that :meth:`reset` writes, shape
+        ``(num_envs, num_joints)`` in rad/s for hinge joints and m/s for
+        slider joints. Columns use the same order as
+        ``state.joint_positions`` and the indices returned by
+        :meth:`find_joints`. Available from ``on_start`` onward. Each read
+        returns a new tensor, so writing into it cannot change a later
+        reset."""
+        ...
+
+    @property
+    def default_pose(self) -> TensorLike:
+        """Base pose that :meth:`reset` writes in the world frame, shape
+        ``(num_envs, 7)`` with columns ``[x, y, z, qw, qx, qy, qz]``:
+        position in m followed by a w-first unit quaternion. Each
+        environment's origin is included. Available from ``on_start``
+        onward. Each read returns a new tensor, so writing into it cannot
+        change a later reset."""
+        ...
+
     @property
     def sensors(self) -> List[Any]:
         """Sensors attached to this robot via :meth:`add_sensor`."""

{simulo_interfaces-0.22.0 → simulo_interfaces-0.23.0}/src/simulo/interfaces/authoring/terrain.py

@@ -59,10 +59,8 @@ class SubTerrainConfig:
             - "hf_pyramid_slope": Height-field pyramid slope
             - "hf_pyramid_slope_inv": Inverted height-field pyramid slope
         proportion: Proportion of terrain grid to use this type (0.0-1.0).
-            Defaults to 1.0 — "use 100% of the grid for this single
-            sub-terrain". Every in-tree caller passes ``proportion=``
-            explicitly (Wave 0 audit verified), so the default-value
-            change from the pre-Wave-B local default of 0.2 is safe.
+            Defaults to 1.0, which uses the whole grid for this single
+            sub-terrain.
         params: Type-specific parameters (e.g., height, step_height, etc.)
 
     Example:

{simulo_interfaces-0.22.0 → simulo_interfaces-0.23.0}/src/simulo/interfaces/platform/__init__.py

@@ -79,6 +79,50 @@ from simulo.interfaces.platform.enums import (
     OutputKind,
     SystemType,
 )
+from simulo.interfaces.platform.evaluation import (
+    ENDED_BY_TASK_TRUNCATION,
+    ENDED_BY_TIME_LIMIT,
+    EPISODES_CSV_CHECK_PREFIX,
+    EPISODES_CSV_COLUMNS_AFTER_CHECKS,
+    EPISODES_CSV_COLUMNS_BEFORE_CHECKS,
+    EVALUATION_ACTIONS,
+    EVALUATION_REPORT_SCHEMA_VERSION,
+    MAX_CHECK_NAME_CHARS,
+    MAX_CHECKS_PER_RULE,
+    MAX_END_CONDITION_NAME_CHARS,
+    MAX_END_CONDITIONS,
+    MAX_EVALUATION_HEADLINE_SUMMARY_CHARS,
+    MAX_FAILURE_GROUPS,
+    MAX_LISTED_EPISODES,
+    MAX_LISTED_SIMULATION_PROBLEMS,
+    MAX_NEXT_LINES,
+    MAX_RECORDED_EPISODES,
+    MAX_RECORDING_PROBLEM_CHARS,
+    CheckCount,
+    ComparisonVerdict,
+    EndingCount,
+    EvaluationActions,
+    EvaluationCheckpoint,
+    EvaluationComparison,
+    EvaluationHeadline,
+    EvaluationReport,
+    EvaluationResult,
+    EvaluationTask,
+    EvaluationTested,
+    FailureGroup,
+    LikelyRange,
+    ListedEpisode,
+    NextLine,
+    NextRule,
+    ReplayFrame,
+    ReplayPose,
+    ReplayTrace,
+    SimulationProblem,
+    SimulationProblemEpisode,
+    end_condition_names_problem,
+    episodes_csv_columns,
+    success_checks_problem,
+)
 from simulo.interfaces.platform.job_events import (
     MAX_PUBLIC_ASSET_FINDING_COUNT,
     MAX_PUBLIC_JOB_EVENT_ATTEMPT,
@@ -241,6 +285,7 @@ from simulo.interfaces.platform.submit import (
     CHECKPOINT_UNAVAILABLE_CODE,
     CHECKPOINT_UPLOAD_MISSING_CODE,
     EXPORTS_ROUTE,
+    INPUT_CHECKPOINT_LATEST_AVAILABLE_META_KEY,
     INPUT_CHECKPOINT_NOT_ALLOWED_CODE,
     JOB_RECORDING_DOWNLOAD_ROUTE_TEMPLATE,
     JOB_RECORDINGS_ROUTE_TEMPLATE,
@@ -419,6 +464,7 @@ __all__ = [
     "WORKER_JOB_CHECKPOINTS_PRESIGN_ROUTE_TEMPLATE",
     "WORKER_JOB_CHECKPOINTS_ROUTE_TEMPLATE",
     "WORKER_CLAIM_INPUT_CHECKPOINT_FIELDS",
+    "INPUT_CHECKPOINT_LATEST_AVAILABLE_META_KEY",
     "WORKER_LEASE_HEADER",
     "MAX_PACKAGE_BYTES",
     "MAX_LOG_CHUNK_BYTES",
@@ -579,6 +625,51 @@ __all__ = [
     "PreviewReportRow",
     "PreviewReportStage",
     "PreviewReport",
+    # evaluation report wire contract — schema version, how an episode ended,
+    # the bounds that keep the report under the result cap, the report shapes,
+    # the replay trace filming reads, and episodes.csv
+    "EVALUATION_REPORT_SCHEMA_VERSION",
+    "MAX_EVALUATION_HEADLINE_SUMMARY_CHARS",
+    "ENDED_BY_TIME_LIMIT",
+    "ENDED_BY_TASK_TRUNCATION",
+    "MAX_END_CONDITION_NAME_CHARS",
+    "MAX_END_CONDITIONS",
+    "MAX_CHECKS_PER_RULE",
+    "MAX_CHECK_NAME_CHARS",
+    "MAX_FAILURE_GROUPS",
+    "MAX_RECORDED_EPISODES",
+    "MAX_RECORDING_PROBLEM_CHARS",
+    "MAX_LISTED_EPISODES",
+    "MAX_LISTED_SIMULATION_PROBLEMS",
+    "MAX_NEXT_LINES",
+    "end_condition_names_problem",
+    "success_checks_problem",
+    "EVALUATION_ACTIONS",
+    "EvaluationActions",
+    "ComparisonVerdict",
+    "NextRule",
+    "SimulationProblem",
+    "EvaluationCheckpoint",
+    "EvaluationTask",
+    "EvaluationTested",
+    "LikelyRange",
+    "EvaluationHeadline",
+    "CheckCount",
+    "EndingCount",
+    "FailureGroup",
+    "EvaluationResult",
+    "EvaluationComparison",
+    "ListedEpisode",
+    "SimulationProblemEpisode",
+    "NextLine",
+    "EvaluationReport",
+    "ReplayPose",
+    "ReplayFrame",
+    "ReplayTrace",
+    "EPISODES_CSV_COLUMNS_BEFORE_CHECKS",
+    "EPISODES_CSV_CHECK_PREFIX",
+    "EPISODES_CSV_COLUMNS_AFTER_CHECKS",
+    "episodes_csv_columns",
     # protocols
     "RuntimeProtocol",
     "AssetProtocol",

{simulo_interfaces-0.22.0 → simulo_interfaces-0.23.0}/src/simulo/interfaces/platform/enums.py

@@ -19,10 +19,10 @@ class JobType(StrEnum):
 
 
 #: Types a user can submit today. Platform-owned jobs use the remaining types.
-USER_SUBMITTABLE_JOB_TYPES = frozenset({JobType.PREVIEW, JobType.TRAIN, JobType.PLAY})
+USER_SUBMITTABLE_JOB_TYPES = frozenset({JobType.PREVIEW, JobType.TRAIN, JobType.EVAL, JobType.PLAY})
 
 #: Declared job types whose user-submission workflows are not available yet.
-UNAVAILABLE_JOB_TYPES = frozenset({JobType.EVAL})
+UNAVAILABLE_JOB_TYPES: frozenset[JobType] = frozenset()
 
 #: Job types whose submission may carry an input policy checkpoint.
 INPUT_CHECKPOINT_JOB_TYPES = frozenset({JobType.TRAIN, JobType.PLAY, JobType.EVAL})

simulo_interfaces-0.23.0/src/simulo/interfaces/platform/evaluation.py

@@ -0,0 +1,451 @@
+"""Evaluation report: what an eval job returns, writes and replays.
+
+An eval job runs a trained checkpoint on numbered episodes and judges each
+one with the eval file's ``@app.success`` function. The runtime returns the
+report as a plain ``dict`` of :class:`EvaluationReport` shape and registers it
+as a ``report`` output; the client renders it, the Console reads it from the
+job result, and the control plane checks its schema version. Every episode's
+detail goes in ``episodes.csv`` (:func:`episodes_csv_columns`), so the report
+stays small whatever the episode count.
+
+The comparison block (``--compare``) is part of v1 from the start, so comparing
+two checkpoints never needs a new schema version. Additive-only: a breaking
+reshape mints ``simulo.evaluation_report.v2``.
+
+Percentages are whole numbers. Times are seconds from the episode's start.
+Checkpoints are written ``policy_x:best`` or ``policy_x:latest``.
+"""
+
+from typing import Final, Literal, NotRequired, Sequence, TypedDict
+
+EVALUATION_REPORT_SCHEMA_VERSION: Final = "simulo.evaluation_report.v1"
+
+# --- How an episode ended ----------------------------------------------------
+#
+# The report's "Why" column and the ``ended_by`` column of ``episodes.csv``
+# hold exactly one of: the name of the task's end condition that ended the
+# episode (a key of the dict ``get_dones()`` returns for ``terminated``),
+# :data:`ENDED_BY_TIME_LIMIT`, or :data:`ENDED_BY_TASK_TRUNCATION`.
+
+ENDED_BY_TIME_LIMIT: Final = "time limit"
+"""The episode reached the time limit: Simulo's step counter reached
+``max_episode_length - 1`` and no end condition was true. Reserved for that
+boundary; it counts as surviving."""
+
+ENDED_BY_TASK_TRUNCATION: Final = "cut short by the task"
+"""The task's ``truncated`` ended the episode before the time limit and no end
+condition was true. Never counts as surviving."""
+
+MAX_END_CONDITION_NAME_CHARS: Final = 64
+"""Longest name ``get_dones()`` may give an end condition."""
+
+# --- Bounds ------------------------------------------------------------------
+#
+# The report is a job result, so it must fit the worker's result cap
+# (``simulo.interfaces.platform.submit.MAX_RESULT_BYTES``, 1 MiB) whatever the
+# episode count. Every list in it is bounded by one of these, and every name in
+# it by a character limit, so its size does not grow with the episodes run.
+# The ``get_dones()`` check enforces the end-condition bounds
+# (:func:`end_condition_names_problem`); the evaluator enforces the check bounds
+# (:func:`success_checks_problem`) and builds every list within its bound.
+
+MAX_END_CONDITIONS: Final = 32
+"""Most end conditions ``get_dones()`` may name. A RESULT's ``endings`` holds
+at most this many plus :data:`ENDED_BY_TIME_LIMIT` and
+:data:`ENDED_BY_TASK_TRUNCATION`."""
+
+MAX_CHECKS_PER_RULE: Final = 16
+"""Most checks one ``@app.success`` function may name with ``task.check()``."""
+
+MAX_CHECK_NAME_CHARS: Final = 64
+"""Longest name ``task.check()`` may give a check."""
+
+MAX_FAILURE_GROUPS: Final = 3
+"""Failure groups a RESULT lists; ``other_failure_combinations`` counts the rest."""
+
+MAX_RECORDED_EPISODES: Final = 10
+"""Episodes an evaluation records."""
+
+MAX_LISTED_EPISODES: Final = 13
+"""Episodes WENT WRONG lists: the recorded ones, then up to three more."""
+
+MAX_LISTED_SIMULATION_PROBLEMS: Final = 10
+"""Episodes SIMULATION PROBLEMS lists; ``simulation_problem_episodes`` counts them all."""
+
+MAX_NEXT_LINES: Final = 4
+"""NEXT lines: up to three rules that match, then the ``--compare`` offer."""
+
+MAX_RECORDING_PROBLEM_CHARS: Final = 512
+"""Longest reason an evaluation can report when requested filming did not complete."""
+
+MAX_EVALUATION_HEADLINE_SUMMARY_CHARS: Final = 512
+"""Longest summary the jobs list may carry for one evaluation, omitted when longer than this.
+
+The full report is available from the evaluation job result.  Policy job lists
+only need a compact rendering input, so this cap prevents one worker result
+from making an otherwise paginated response unbounded.
+"""
+
+_RESERVED_ENDINGS: Final = {
+    ENDED_BY_TIME_LIMIT: "an episode that ran to the time limit",
+    ENDED_BY_TASK_TRUNCATION: "an episode your truncated ended before the time limit",
+}
+
+
+def _name_problem(source: str, noun: str, example: str, limit: int, name: object) -> str | None:
+    if not isinstance(name, str):
+        return f'{source} {noun} named {name!r}; name each one with a string, such as "{example}".'
+    if not name:
+        return f'{source} {noun} with an empty name; name each one, such as "{example}".'
+    if name != name.strip():
+        return f'{source} "{name}", which starts or ends with a space; name it "{name.strip()}".'
+    if len(name) > limit:
+        return f'{source} "{name}", a name of {len(name)} characters; names can be at most {limit} characters.'
+    return None
+
+
+def end_condition_names_problem(names: Sequence[object]) -> str | None:
+    """What is wrong with the end-condition names ``get_dones()`` returned, or ``None``.
+
+    The message names the condition and says how to fix the task. Each name is
+    a non-empty string of at most :data:`MAX_END_CONDITION_NAME_CHARS`
+    characters with no leading or trailing space, and not one of the two words
+    the report uses for an episode no condition ended (in any case); there are
+    at most :data:`MAX_END_CONDITIONS` of them.
+    """
+
+    if len(names) > MAX_END_CONDITIONS:
+        return (
+            f"get_dones() returned {len(names)} end conditions; a task can name at most "
+            f"{MAX_END_CONDITIONS}, so combine some of them."
+        )
+    for name in names:
+        source = "get_dones() returned"
+        problem = _name_problem(source, "an end condition", "pole fell", MAX_END_CONDITION_NAME_CHARS, name)
+        if problem is None and isinstance(name, str) and name.casefold() in _RESERVED_ENDINGS:
+            meaning = _RESERVED_ENDINGS[name.casefold()]
+            problem = f'{source} "{name}", which the report uses for {meaning}; give the condition another name.'
+        if problem is not None:
+            return problem
+    return None
+
+
+def success_checks_problem(function: str, names: Sequence[object]) -> str | None:
+    """What is wrong with the checks the ``@app.success`` function ``function`` named, or ``None``.
+
+    ``names`` are its ``task.check()`` names in call order. Each is a non-empty
+    string of at most :data:`MAX_CHECK_NAME_CHARS` characters with no leading or
+    trailing space; there are at most :data:`MAX_CHECKS_PER_RULE` of them.
+    """
+
+    if len(names) > MAX_CHECKS_PER_RULE:
+        return (
+            f"{function}() named {len(names)} checks with task.check(); a success rule can name at most "
+            f"{MAX_CHECKS_PER_RULE}, so combine some of them."
+        )
+    for name in names:
+        problem = _name_problem("task.check() got", "a check", "pole upright", MAX_CHECK_NAME_CHARS, name)
+        if problem is not None:
+            return problem
+    return None
+
+
+EvaluationActions = Literal["best", "sampled"]
+"""``best`` is the policy's best-guess action, what you would deploy;
+``sampled`` keeps training's random variation on, as a diagnostic."""
+
+EVALUATION_ACTIONS: Final = ("best", "sampled")
+
+ComparisonVerdict = Literal["did_better", "did_worse", "no_clear_difference"]
+"""How the ``--policy`` checkpoint did against the ``--compare`` checkpoint,
+judged on the episodes whose outcome changed (McNemar's exact test, p < 0.05)."""
+
+NextRule = Literal[
+    "simulation_problems",
+    "never_learned",
+    "reward_pays_for_failure",
+    "one_check_missed",
+    "more_episodes",
+    "watch_worst_failure",
+    "all_succeeded",
+    "compare_latest",
+]
+"""Which rule produced a NEXT line. The first seven are checked in this order and
+NEXT prints up to three that match: simulation problems; a likely range whose
+top is under 10%; the failures' median total reward at least the successes';
+the largest failure group missing exactly one check; a likely range wider than
+15 points or crossing ``--require-success``; some failures, when the second
+rule did not match; every episode succeeded. ``compare_latest`` is the ``--compare policy_x:latest`` offer after them."""
+
+SimulationProblem = Literal["reward_not_finite", "observation_not_finite", "observation_out_of_bounds"]
+"""What made a step unusable; an episode with one counts as a failure."""
+
+
+class EvaluationCheckpoint(TypedDict):
+    """One checkpoint that was evaluated, from its sidecar."""
+
+    reference: str
+    """``policy_x:best`` or ``policy_x:latest``."""
+    policy_id: str
+    kind: Literal["best", "latest"]
+    iteration: int
+    training_reward: float | None
+    digest_sha256: str
+    """Digest of the checkpoint file, so a report names the exact weights."""
+
+
+class EvaluationTask(TypedDict):
+    """The task the episodes ran, against the one the checkpoint was trained on."""
+
+    name: str
+    """The task class name."""
+    file: str
+    digest: str
+    """Hash of today's ``task.py``."""
+    trained_digest: str | None
+    """Hash of ``task.py`` when the checkpoint was trained; ``None`` when unknown."""
+    changed: bool
+    """The task-changed flag: the class name or the digest differs from training's."""
+    end_conditions: list[str]
+    """The names ``get_dones()`` gives its end conditions, in the task's order;
+    empty for a task that never ends an episode early. At most
+    :data:`MAX_END_CONDITIONS`."""
+
+
+class EvaluationTested(TypedDict):
+    """WHAT WAS TESTED."""
+
+    checkpoint: EvaluationCheckpoint
+    task: EvaluationTask
+    first_episode: int
+    last_episode: int
+    """Episode numbers run, inclusive. Each number fixes its starting state."""
+    num_envs: int
+    actions: EvaluationActions
+    success_function: str
+    """The ``@app.success`` function's name, such as ``balanced``."""
+    success_file: str
+    checks: list[str]
+    """Every ``task.check()`` name, in the order the rule declares them; at most
+    :data:`MAX_CHECKS_PER_RULE`."""
+
+
+class LikelyRange(TypedDict):
+    """95% Wilson score interval, low rounded down and high rounded up."""
+
+    low_percent: int
+    high_percent: int
+
+
+class EvaluationHeadline(TypedDict):
+    """Compact result projection served on an evaluation job record."""
+
+    summary: str
+    """The report's first RESULT line, omitted when longer than :data:`MAX_EVALUATION_HEADLINE_SUMMARY_CHARS`."""
+    episodes: int
+    succeeded: int
+    likely_range: LikelyRange
+
+
+class CheckCount(TypedDict):
+    """How many episodes passed one check on its own."""
+
+    name: str
+    passed: int
+
+
+class EndingCount(TypedDict):
+    """How many episodes ended one way."""
+
+    ended_by: str
+    """A condition name, :data:`ENDED_BY_TIME_LIMIT` or :data:`ENDED_BY_TASK_TRUNCATION`."""
+    episodes: int
+
+
+class FailureGroup(TypedDict):
+    """Failures grouped by exactly which checks were "no"."""
+
+    missed: list[str]
+    """Check names, in rule order; empty when the rule failed with every check passing."""
+    episodes: int
+
+
+class EvaluationResult(TypedDict):
+    """RESULT for one checkpoint."""
+
+    checkpoint: str
+    """The checkpoint's ``reference``."""
+    episodes: int
+    succeeded: int
+    likely_range: LikelyRange
+    checks: list[CheckCount]
+    """One entry per check, in rule order."""
+    failure_groups: list[FailureGroup]
+    """Most common first, at most :data:`MAX_FAILURE_GROUPS`."""
+    other_failure_combinations: int
+    """How many further groups exist beyond ``failure_groups``. Their episodes
+    are the failures ``failure_groups`` doesn't hold, so the listed groups
+    account for every failure exactly when this is 0."""
+    endings: list[EndingCount]
+    """Every episode counted by how it ended, most common first."""
+    simulation_problem_episodes: int
+
+
+class EvaluationComparison(TypedDict):
+    """The ``--compare`` verdict: the same episode numbers, run by both checkpoints."""
+
+    verdict: ComparisonVerdict
+    p_value: float
+    compared_with: EvaluationCheckpoint
+    """The ``--compare`` checkpoint."""
+    compared_result: EvaluationResult
+    """The ``--compare`` checkpoint's own RESULT."""
+    improved: int
+    """Episodes that failed with ``compared_with`` and succeeded with the ``--policy`` checkpoint."""
+    regressed: int
+    """Episodes that succeeded with ``compared_with`` and failed with the ``--policy`` checkpoint."""
+
+
+class ListedEpisode(TypedDict):
+    """One episode in WENT WRONG: a failure, or a recorded success when all succeeded."""
+
+    episode: int
+    success: bool
+    ended_by: str
+    """A condition name, :data:`ENDED_BY_TIME_LIMIT` or :data:`ENDED_BY_TASK_TRUNCATION`."""
+    ended_at_s: float
+    total_reward: float
+    checks: dict[str, bool]
+    """Each check's value on the ending step, keyed by name in rule order."""
+    simulation_problem: SimulationProblem | None
+    recording: str | None
+    """The recording name for ``simulo recordings JOB <name>``, such as ``episode-64``;
+    ``None`` when the episode was not recorded."""
+
+
+class SimulationProblemEpisode(TypedDict):
+    """One episode in SIMULATION PROBLEMS, at the first step that went wrong."""
+
+    episode: int
+    problem: SimulationProblem
+    at_s: float
+
+
+class NextLine(TypedDict):
+    """One NEXT line: what to do, and the command that does it."""
+
+    rule: NextRule
+    text: str
+    command: str | None
+
+
+class EvaluationReport(TypedDict):
+    """The whole report, as returned by the eval job body."""
+
+    report_schema_version: Literal["simulo.evaluation_report.v1"]
+    summary: str
+    """RESULT's first line, such as ``87 of 100 episodes succeeded. Likely range: 79% to 93%.``"""
+    task_changed: bool
+    """Same as ``tested.task.changed``; when true, ``notice`` leads the report."""
+    notice: str | None
+    """The first line when the task changed since training, such as
+    ``task.py has changed since policy_x was trained``; otherwise ``None``."""
+    runtime_version: str
+    """The ``simulo-backend`` version the evaluation ran on."""
+    tested: EvaluationTested
+    result: EvaluationResult
+    """The ``--policy`` checkpoint's RESULT."""
+    comparison: EvaluationComparison | None
+    """Set only for a ``--compare`` evaluation."""
+    failures: int
+    """How many episodes failed; ``listed_episodes`` holds a selection of them."""
+    recorded: int
+    """How many episodes have a recording, at most :data:`MAX_RECORDED_EPISODES`."""
+    listed_episodes: list[ListedEpisode]
+    """The recorded episodes in the order they were chosen, then up to three more
+    failures by lowest total reward (ties by episode number); at most
+    :data:`MAX_LISTED_EPISODES`."""
+    simulation_problems: list[SimulationProblemEpisode]
+    """The first :data:`MAX_LISTED_SIMULATION_PROBLEMS` episodes with a
+    simulation problem, in episode order."""
+    next: list[NextLine]
+    """At most :data:`MAX_NEXT_LINES`."""
+    recording_problem: NotRequired[str]
+    """Why requested filming did not complete, at most
+    :data:`MAX_RECORDING_PROBLEM_CHARS` characters. Omitted when filming succeeds
+    and by reports written before this additive v1 field existed."""
+
+
+# --- Replay traces (filming) -------------------------------------------------
+#
+# While an evaluation runs, each counted episode buffers the state needed to
+# replay it. The filming pass replays a kept trace frame by frame by setting
+# that state, with no physics, and films it.
+
+
+class ReplayPose(TypedDict):
+    """A world-frame pose: position in metres, orientation as a (w, x, y, z) quaternion."""
+
+    position: list[float]
+    orientation_wxyz: list[float]
+
+
+class ReplayFrame(TypedDict):
+    """The robot's and the scene's state at one moment: the start, then after each policy step."""
+
+    time_s: float
+    joint_positions: list[float]
+    """One value per joint, in ``ReplayTrace.joint_names`` order: radians for a hinge, metres for a slider."""
+    joint_velocities: list[float]
+    root_pose: ReplayPose
+    object_poses: dict[str, ReplayPose]
+    """The scene's moving objects other than the robot, keyed by their scene path."""
+
+
+class ReplayTrace(TypedDict):
+    """One episode, replayable without physics."""
+
+    episode: int
+    checkpoint: str
+    joint_names: list[str]
+    step_dt: float
+    """Seconds between frames."""
+    source_env_origin: list[float]
+    """The source environment's world-frame xyz origin as three finite metres.
+
+    Filming translates world-frame poses from this origin into its destination
+    environment.
+    """
+    frames: list[ReplayFrame]
+    """The first frame is the episode's starting state, ``time_s == 0``."""
+
+
+# --- episodes.csv ------------------------------------------------------------
+#
+# One row per episode. Booleans are ``true``/``false``. ``sim_problem`` holds a
+# ``SimulationProblem`` value and ``recording`` a recording name, each empty when
+# there is none. ``first_observation`` is the policy's observation on the
+# episode's first step, as a JSON list.
+
+EPISODES_CSV_COLUMNS_BEFORE_CHECKS: Final = (
+    "episode",
+    "seed",
+    "checkpoint",
+    "success",
+    "ended_by",
+    "ended_at_s",
+    "total_reward",
+    "sim_problem",
+)
+EPISODES_CSV_CHECK_PREFIX: Final = "check:"
+EPISODES_CSV_COLUMNS_AFTER_CHECKS: Final = ("first_observation", "recording")
+
+
+def episodes_csv_columns(check_names: Sequence[str]) -> tuple[str, ...]:
+    """The ``episodes.csv`` header for a rule with these checks, in rule order."""
+
+    return (
+        *EPISODES_CSV_COLUMNS_BEFORE_CHECKS,
+        *(EPISODES_CSV_CHECK_PREFIX + name for name in check_names),
+        *EPISODES_CSV_COLUMNS_AFTER_CHECKS,
+    )

{simulo_interfaces-0.22.0 → simulo_interfaces-0.23.0}/src/simulo/interfaces/platform/manifest.py

@@ -96,11 +96,11 @@ class Manifest:
       source regardless of which job or args were submitted; used for
       reproducibility / source caching.
     * ``package_id`` — the *submission* identity: a ``sha256`` over
-      ``source_digest + job_name + canonical_json(args)``. It is also the
-      package directory name. Two submits of the same source with a
-      different job or args get a different ``package_id`` (a distinct
-      package dir), so one submission never clobbers another's recorded
-      ``args``.
+      ``source_digest + entrypoint.file + job_name + canonical_json(args)``.
+      It is also the package directory name. Two submits of the same source
+      with a different entry file, job, or args get a different ``package_id``
+      (a distinct package dir), so one submission never clobbers another's
+      recorded entrypoint or ``args``.
 
     ``manifest_version`` defaults to **2**. Every entry in ``jobs`` carries a
     required ``"type"`` key holding a :class:`~simulo.interfaces.platform.enums.JobType`
@@ -109,7 +109,8 @@ class Manifest:
 
     ⚠️ **Wire stability:** :meth:`to_json` bytes feed the package archive,
     whose sha256 the control plane pins per ``package_id`` — while
-    ``package_id`` itself hashes only ``(source_digest, job_name, args)``.
+    ``package_id`` itself hashes only
+    ``(source_digest, entrypoint.file, job_name, args)``.
     Any change to this dataclass's fields or serialisation therefore shifts
     ``archive_sha256`` under an UNCHANGED ``package_id``, which is exactly
     the server's ``package_conflict`` 409 for every user resubmitting an

{simulo_interfaces-0.22.0 → simulo_interfaces-0.23.0}/src/simulo/interfaces/platform/runs.py

@@ -193,6 +193,7 @@ from typing import Any, Optional
 from simulo.interfaces.ids import JobId, JobPublicId, PackageId, parse_job_public_id
 from simulo.interfaces.platform._payload import known_payload_fields
 from simulo.interfaces.platform.enums import JobStatus, JobType
+from simulo.interfaces.platform.evaluation import EvaluationHeadline
 from simulo.interfaces.platform.outputs import JobOutputsSummary, OutputFlag
 
 #: Default TCP port for the jobs API. Locally this is where ``simulo-backend
@@ -413,6 +414,16 @@ class JobRecord:
     input_checkpoint: Optional[str] = None
     """The policy checkpoint this job started from, or ``None`` for a fresh job."""
 
+    compare_checkpoint: Optional[str] = None
+    """The checkpoint an evaluation compares against, or ``None`` when it evaluates one policy."""
+
+    evaluation: Optional[EvaluationHeadline] = None
+    """Compact result for an evaluation job, or ``None`` for another job or an
+    evaluation whose report is not available. Only the policy job list fills
+    this field. The full evaluation report stays on the job result endpoint;
+    this list-safe projection carries only its summary, episode count,
+    successes, and likely success-rate range."""
+
     tags: tuple[str, ...] = ()
     """Immutable, normalized job tags in first-seen order."""
 

{simulo_interfaces-0.22.0 → simulo_interfaces-0.23.0}/src/simulo/interfaces/platform/submit.py

@@ -104,6 +104,11 @@ platform-standard shape: ``{"error": {"code": ..., "message": ..., "request_id":
   checkpoint belongs to one execution. Consumers therefore keep
   ``input_checkpoint`` outside ``args`` and outside the package manifest.
 
+  An evaluation submission may additionally carry ``compare_checkpoint`` at
+  the same top level. It uses the same policy checkpoint grammar and resolves
+  inside the selected organization. The two fields identify their roles: the
+  ``input_checkpoint`` is evaluated and ``compare_checkpoint`` is the baseline.
+
   **App name (live-view wave, additive):** the request body
   optionally carries ``app_name`` (``str``) — the submitting ``App``'s
   declared name, display context the platform's live viewer shows next to
@@ -502,6 +507,11 @@ WORKER_CLAIM_INPUT_CHECKPOINT_FIELDS = (
     "checkpoint_meta",
 )
 
+#: Claim-time metadata fact stating whether the source policy currently has a
+#: ``latest`` checkpoint. The control plane owns this availability snapshot;
+#: workers carry it through the installed sidecar to ``PolicyCheckpoint.meta``.
+INPUT_CHECKPOINT_LATEST_AVAILABLE_META_KEY = "input_checkpoint_latest_available"
+
 #: Request header carrying the claim's ``lease_id`` ownership token on every
 #: job-scoped worker call (heartbeat / logs / events / complete). A header — not a body
 #: field — because the logs route's body is raw bytes and heartbeat has no

{simulo_interfaces-0.22.0 → simulo_interfaces-0.23.0}/src/simulo/interfaces/runtime/components.py

@@ -87,6 +87,9 @@ class TerminationConditionProtocol(Protocol):
     mirror of ``simulo.core.terminations.TerminationCondition``.
     """
 
+    name: Optional[str]
+    """Optional plain-language name used to report this end condition."""
+
     def initialize(self, robot: RobotProtocol, device: str, num_envs: int) -> None:
         """One-time setup: cache the ``robot`` handle and size buffers for ``num_envs`` on ``device``."""
         ...

{simulo_interfaces-0.22.0 → simulo_interfaces-0.23.0}/src/simulo/interfaces/runtime/env.py

@@ -90,12 +90,24 @@ class LearningEnvProtocol(Protocol):
     def step(self, actions: TensorLike) -> Tuple[TensorLike, TensorLike, TensorLike, TensorLike, Dict[str, Any]]:
         """Advance one step with ``actions`` ``(num_envs, action_dim)``.
 
+        The environment passes a copy of ``actions`` to the task before
+        physics advances. Tasks can modify that copy in place; the original
+        batch remains available to recording as the policy's choice.
+
         Returns ``(observations, rewards, terminated, truncated, info)`` — the
         gym-style 5-tuple, each tensor batched over ``num_envs``. Environments
         that finish are auto-reset by the implementation.
         """
         ...
 
+    def seeded_reset(self, env_id: int, episode_number: int) -> None:
+        """Reset one environment after seeding global generators for an episode."""
+        ...
+
+    def add_pre_reset_hook(self, hook: Any) -> None:
+        """Register code to run after dones are collected and before reset."""
+        ...
+
     def close(self) -> None:
         """Shut the simulation down and release its resources."""
         ...

{simulo_interfaces-0.22.0 → simulo_interfaces-0.23.0}/src/simulo/interfaces/runtime/task.py

@@ -1,6 +1,6 @@
 """``TaskProtocol`` — structural mirror of ``simulo.core.task.Task`` (Part A)."""
 
-from typing import Any, Protocol, Tuple, runtime_checkable
+from typing import Any, Dict, Protocol, Tuple, runtime_checkable
 
 # A REAL import, deliberately not TYPE_CHECKING-guarded: the guarded form left
 # ``typing.get_type_hints(TaskProtocol.build)`` raising ``NameError`` at
@@ -24,6 +24,11 @@ class TaskProtocol(Protocol):
 
     Structural mirror of ``simulo.core.task.Task``, the abstract base your task
     subclasses on the worker.
+
+    Each environment step calls ``on_pre_physics_step(actions)``, advances
+    physics, then calls ``on_post_physics_step()`` before collecting dones,
+    rewards, observations, and a pre-reset recording snapshot. It resets
+    environments whose episodes ended after that snapshot.
     """
 
     observation_dim: int
@@ -54,6 +59,25 @@ class TaskProtocol(Protocol):
         """
         ...
 
+    def on_pre_physics_step(self, actions: TensorLike) -> None:
+        """Process one policy action batch before physics advances.
+
+        ``LearningEnv.step`` gives this hook a copy of the policy actions, so
+        tasks may safely scale or otherwise modify ``actions`` in place. The
+        environment keeps the original batch for recording what the policy
+        chose.
+        """
+        return None
+
+    def on_post_physics_step(self) -> None:
+        """Run once after physics advances and before dones, rewards, and observations.
+
+        Use this hook for work that must run exactly once per environment step,
+        or for an expensive value shared by several later task methods. Values
+        cached here must also be set for reset environments in ``reset_idx``.
+        """
+        return None
+
     def get_observations(self) -> TensorLike:
         """Return the observation batch, shape ``(num_envs, observation_dim)``."""
         ...
@@ -62,16 +86,21 @@ class TaskProtocol(Protocol):
         """Return the per-step reward for every environment, shape ``(num_envs,)``."""
         ...
 
-    def get_dones(self) -> Tuple[TensorLike, TensorLike]:
-        """Return ``(terminated, truncated)`` boolean batches, each shape ``(num_envs,)``.
+    def get_dones(self) -> Tuple[Dict[str, TensorLike], TensorLike]:
+        """Return named termination batches and one truncated batch.
 
-        ``terminated`` marks episodes ended by the task itself (success/failure);
-        ``truncated`` marks episodes cut off by a time limit.
+        Each ``terminated`` value marks where the task ended the episode,
+        whatever the reason. ``truncated`` marks episodes cut off by a time
+        limit.
         """
         ...
 
     def apply_actions(self, actions: TensorLike) -> None:
-        """Apply the policy's action batch, shape ``(num_envs, action_dim)``, to the sim."""
+        """Apply a copy of the policy's action batch to the sim.
+
+        ``actions`` has shape ``(num_envs, action_dim)``. It may be modified in
+        place without changing the policy action batch the environment records.
+        """
         ...
 
     def reset_idx(self, env_ids: TensorLike) -> None:

{simulo_interfaces-0.22.0 → simulo_interfaces-0.23.0}/src/simulo/interfaces/runtime/trainer.py

@@ -1,7 +1,7 @@
 """``TrainerProtocol`` — structural mirror of ``simulo.core.trainer.Trainer`` (Part A).
 
-The contract is deliberately behavioral — ``train`` / ``evaluate`` / ``save``
-/ ``load`` and nothing else (re-affirmed at the Wave C4 mirror completion; the
+The contract is deliberately behavioral — ``train`` / ``save`` / ``load`` and
+nothing else (re-affirmed at the Wave C4 mirror completion; the
 executable half of this record is the documented-omissions allowlist in the
 backend suite's conformance engine): ``env`` / ``device`` / ``seed`` /
 ``log_dir`` are conveniences the base ``simulo.core.trainer.Trainer.__init__``
@@ -20,12 +20,12 @@ user-extensible ABC) does not guarantee. Implementations that CAN export (today:
 consumers feature-detect with ``isinstance(trainer, PolicyExporterProtocol)``.
 """
 
-from typing import Any, Dict, Optional, Protocol, runtime_checkable
+from typing import Any, Dict, Protocol, runtime_checkable
 
 
 @runtime_checkable
 class TrainerProtocol(Protocol):
-    """The training-orchestration contract — run, evaluate, and persist a training job.
+    """The training-orchestration contract — run and persist a training job.
 
     A trainer owns the RL loop: it drives an environment, updates the policy, and
     reads/writes checkpoints. Structural mirror of ``simulo.core.trainer.Trainer``.
@@ -39,14 +39,6 @@ class TrainerProtocol(Protocol):
         """
         ...
 
-    def evaluate(self, checkpoint: Optional[str] = ..., num_episodes: int = ...) -> Dict[str, Any]:
-        """Roll out the current (or ``checkpoint``-loaded) policy without learning.
-
-        Returns evaluation metrics such as mean episode return over
-        ``num_episodes`` episodes.
-        """
-        ...
-
     def save(self, path: str) -> None:
         """Write the trainer's full state (policy + optimizer) to ``path``."""
         ...

{simulo_interfaces-0.22.0 → simulo_interfaces-0.23.0/src/simulo_interfaces.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: simulo-interfaces
-Version: 0.22.0
+Version: 0.23.0
 Summary: Typed Python contracts for the Simulo cloud robotics SDK and managed simulation runtime.
 Author-email: Simulo Team <team@simulo.ai>
 License: BSD-3-Clause

{simulo_interfaces-0.22.0 → simulo_interfaces-0.23.0}/src/simulo_interfaces.egg-info/SOURCES.txt

@@ -32,6 +32,7 @@ src/simulo/interfaces/platform/callbacks.py
 src/simulo/interfaces/platform/debug.py
 src/simulo/interfaces/platform/domain.py
 src/simulo/interfaces/platform/enums.py
+src/simulo/interfaces/platform/evaluation.py
 src/simulo/interfaces/platform/job_events.py
 src/simulo/interfaces/platform/manifest.py
 src/simulo/interfaces/platform/onnx_bundle.py