nomad-harness 0.1.0.dev0__py3-none-any.whl

nomad/__init__.py

@@ -0,0 +1,57 @@
+"""Nomad: an action runtime for frontier models in physical environments.
+
+Importing this package must stay lightweight: it never imports model-provider
+SDKs, simulators, or the adapter subpackages (``nomad.models``,
+``nomad.embodiments``). ``tests/test_import_hygiene.py`` enforces this.
+"""
+
+from nomad.agent import Agent
+from nomad.core import (
+    Action,
+    ActionResult,
+    ActionStatus,
+    AgentContext,
+    Embodiment,
+    EmbodimentManifest,
+    FeedbackCode,
+    ModelProvider,
+    ModelResponse,
+    Observation,
+    ObservationMode,
+    OutcomeStatus,
+    Proposal,
+    RunConfig,
+    RunMetrics,
+    RunResult,
+    RunStatus,
+    StopReason,
+    TaskOutcome,
+)
+from nomad.runtime import Runtime
+
+__version__ = "0.1.0.dev0"
+
+__all__ = [
+    "Action",
+    "ActionResult",
+    "ActionStatus",
+    "Agent",
+    "AgentContext",
+    "Embodiment",
+    "EmbodimentManifest",
+    "FeedbackCode",
+    "ModelProvider",
+    "ModelResponse",
+    "Observation",
+    "ObservationMode",
+    "OutcomeStatus",
+    "Proposal",
+    "RunConfig",
+    "RunMetrics",
+    "RunResult",
+    "RunStatus",
+    "Runtime",
+    "StopReason",
+    "TaskOutcome",
+    "__version__",
+]

nomad/agent.py

@@ -0,0 +1,32 @@
+"""User-facing entry point: bind a model, an embodiment, and a run configuration."""
+
+from __future__ import annotations
+
+import threading
+
+from nomad.context import DEFAULT_INSTRUCTIONS
+from nomad.core.protocols import Embodiment, ModelProvider
+from nomad.core.run import RunConfig, RunResult
+from nomad.runtime import Runtime
+
+
+class Agent:
+    """Runs goals on an embodiment with a model, under one :class:`RunConfig`.
+
+    The agent does not own the embodiment's lifetime: open and close it with a
+    ``with`` block (or explicitly) around one or more ``run`` calls.
+    """
+
+    def __init__(
+        self,
+        model: ModelProvider,
+        embodiment: Embodiment,
+        config: RunConfig | None = None,
+        *,
+        instructions_template: str = DEFAULT_INSTRUCTIONS,
+    ) -> None:
+        self.config = config or RunConfig()
+        self.runtime = Runtime(model, embodiment, instructions_template=instructions_template)
+
+    def run(self, goal: str, *, cancel: threading.Event | None = None) -> RunResult:
+        return self.runtime.run(goal, self.config, cancel=cancel)

nomad/budget.py

@@ -0,0 +1,135 @@
+"""Run budgets and the counters behind ``metrics.json``.
+
+:class:`BudgetTracker` is the single place where loop events are counted. The
+runtime records what happened; the tracker decides whether any budget is
+exhausted and produces the final :class:`~nomad.core.run.RunMetrics`.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+
+from nomad.core.codes import ActionStatus, ModelResponseStatus, StopReason
+from nomad.core.model_io import ModelResponse
+from nomad.core.results import ActionResult
+from nomad.core.run import RunConfig, RunMetrics
+
+
+class BudgetTracker:
+    """Counts decisions, provider calls, tokens, failures, and elapsed time.
+
+    Consecutive failures are any decisions that did not end in a completed
+    action (provider errors, rejections, execution failures); a completed
+    action resets the streak. Token budgets apply only while every response has
+    reported usage: once any call omits it, totals become unknown and the
+    token budget can no longer be enforced.
+    """
+
+    def __init__(self, config: RunConfig, *, clock: Callable[[], float] = time.monotonic) -> None:
+        self._config = config
+        self._clock = clock
+        self._started = clock()
+
+        self.decisions = 0
+        self.provider_calls = 0
+        self.provider_errors = 0
+        self.rejections = 0
+        self.actions_completed = 0
+        self.execution_failures = 0
+        self.consecutive_failures = 0
+        self.model_latency_s = 0.0
+        self.sim_time_s = 0.0
+        self._input_tokens: int | None = 0
+        self._output_tokens: int | None = 0
+
+    # --- recording ----------------------------------------------------------------
+
+    def record_decision(self) -> None:
+        self.decisions += 1
+
+    def record_model_response(self, response: ModelResponse) -> None:
+        self.provider_calls += 1
+        self.model_latency_s += response.latency_s
+        usage = response.usage
+        self._input_tokens = _add(self._input_tokens, usage.input_tokens if usage else None)
+        self._output_tokens = _add(self._output_tokens, usage.output_tokens if usage else None)
+        if response.status is not ModelResponseStatus.OK:
+            self.provider_errors += 1
+            self.consecutive_failures += 1
+
+    def record_rejection(self) -> None:
+        """A proposal that never became a dispatched action (unparseable or invalid)."""
+        self.rejections += 1
+        self.consecutive_failures += 1
+
+    def record_no_progress(self) -> None:
+        """A decision that made no progress without failing, e.g. an unconfirmed finish request."""
+        self.consecutive_failures += 1
+
+    def record_result(self, result: ActionResult) -> None:
+        if result.status is ActionStatus.COMPLETED:
+            self.actions_completed += 1
+            self.consecutive_failures = 0
+            return
+        if result.status is ActionStatus.REJECTED:
+            self.rejections += 1
+        else:
+            self.execution_failures += 1
+        self.consecutive_failures += 1
+
+    def record_sim_time(self, sim_time_s: float) -> None:
+        self.sim_time_s = sim_time_s
+
+    # --- queries ------------------------------------------------------------------
+
+    @property
+    def wall_time_s(self) -> float:
+        return self._clock() - self._started
+
+    @property
+    def total_tokens(self) -> int | None:
+        if self._input_tokens is None or self._output_tokens is None:
+            return None
+        return self._input_tokens + self._output_tokens
+
+    def exhausted(self) -> StopReason | None:
+        """The first exhausted budget, or ``None`` if the loop may continue."""
+        c = self._config
+        if self.decisions >= c.max_decisions:
+            return StopReason.MAX_DECISIONS
+        if self.wall_time_s >= c.max_wall_time_s:
+            return StopReason.MAX_WALL_TIME
+        if c.max_sim_time_s is not None and self.sim_time_s >= c.max_sim_time_s:
+            return StopReason.MAX_SIM_TIME
+        if c.max_provider_calls is not None and self.provider_calls >= c.max_provider_calls:
+            return StopReason.MAX_PROVIDER_CALLS
+        tokens = self.total_tokens
+        if c.max_total_tokens is not None and tokens is not None and tokens >= c.max_total_tokens:
+            return StopReason.MAX_TOKENS
+        if self.consecutive_failures >= c.max_consecutive_failures:
+            return StopReason.MAX_CONSECUTIVE_FAILURES
+        return None
+
+    def metrics(self) -> RunMetrics:
+        known = self.provider_calls > 0
+        return RunMetrics(
+            decisions=self.decisions,
+            provider_calls=self.provider_calls,
+            provider_errors=self.provider_errors,
+            rejections=self.rejections,
+            actions_completed=self.actions_completed,
+            execution_failures=self.execution_failures,
+            input_tokens=self._input_tokens if known else None,
+            output_tokens=self._output_tokens if known else None,
+            model_latency_s=self.model_latency_s,
+            wall_time_s=max(0.0, self.wall_time_s),
+            sim_time_s=self.sim_time_s,
+        )
+
+
+def _add(total: int | None, value: int | None) -> int | None:
+    # Unknown is sticky: a sum that includes an unreported value is unknown.
+    if total is None or value is None:
+        return None
+    return total + value

nomad/context.py

@@ -0,0 +1,89 @@
+"""Builds the canonical, provider-agnostic context for each model decision."""
+
+from __future__ import annotations
+
+from collections import deque
+from typing import Any
+
+from nomad.core.codes import FeedbackCode, ObservationMode
+from nomad.core.manifest import EmbodimentManifest, proposal_json_schema
+from nomad.core.model_io import AgentContext, HistoryEntry
+from nomad.core.observations import Observation
+
+DEFAULT_INSTRUCTIONS = """\
+You control a robot embodiment ({embodiment_id}: {robot}, task {task}) one action at a time.
+After each action you receive a new observation and feedback on what actually executed.
+
+Conventions:
+- Poses are in the right-handed '{pose_frame}' frame: meters, radians, quaternions as [x, y, z, w].
+- move_ee: translation_m is the TOTAL end-effector displacement for this action, not a velocity.
+  rotation_vector_rad is an axis-angle vector (its length is the angle), applied in the frame.
+  It is not Euler roll/pitch/yaw.
+- set_gripper: closure 0 is fully open, 1 is fully closed. It is a setting, not a force.
+- duration_s is the maximum execution time; motion that does not converge reports a timeout.
+
+Rules:
+- Respond with exactly one action per decision, matching the provided schema.
+- Requests outside the limits are rejected, not clipped; read the feedback and adjust.
+- Set request_finish to true when you believe the task is complete. An independent check
+  decides success; requesting finish does not make the task succeed.
+"""
+
+
+class ContextBuilder:
+    """Turns observations and feedback into :class:`AgentContext` records.
+
+    The builder is the only place where model-visible state is decided:
+
+    - ``privileged`` simulator state is removed unless ``mode`` is
+      ``privileged_state``;
+    - history keeps the last ``history_window`` decisions; a window of 0
+      disables both history and last-error feedback (the feedback-off ablation);
+    - ``last_error`` is the message of the most recent unsuccessful decision and
+      clears after a successful action.
+    """
+
+    def __init__(
+        self,
+        manifest: EmbodimentManifest,
+        *,
+        mode: ObservationMode,
+        history_window: int,
+        instructions_template: str = DEFAULT_INSTRUCTIONS,
+    ) -> None:
+        self._mode = mode
+        self._feedback_enabled = history_window > 0
+        self._history: deque[HistoryEntry] = deque(maxlen=history_window)
+        self._last_error: str | None = None
+        self.schema: dict[str, Any] = proposal_json_schema(manifest)
+        self.instructions = instructions_template.format(
+            embodiment_id=manifest.embodiment_id,
+            robot=manifest.robot,
+            task=manifest.task,
+            pose_frame=manifest.pose_frame,
+        )
+
+    def record(self, entry: HistoryEntry) -> None:
+        self._history.append(entry)
+        if entry.code is FeedbackCode.OK:
+            self._last_error = None
+        else:
+            detail = f": {entry.message}" if entry.message else ""
+            self._last_error = f"step {entry.step}: {entry.code}{detail}"
+
+    def build(self, goal: str, observation: Observation) -> AgentContext:
+        visible = observation
+        if (
+            self._mode is not ObservationMode.PRIVILEGED_STATE
+            and observation.privileged is not None
+        ):
+            visible = observation.model_copy(update={"privileged": None})
+        return AgentContext(
+            goal=goal,
+            instructions=self.instructions,
+            mode=self._mode,
+            observation=visible,
+            action_schema=self.schema,
+            history=tuple(self._history),
+            last_error=self._last_error if self._feedback_enabled else None,
+        )

nomad/core/__init__.py

@@ -0,0 +1,78 @@
+"""Typed contracts and protocols shared by the runtime and every adapter."""
+
+from nomad.core.actions import (
+    Action,
+    ActionCall,
+    MoveEEArgs,
+    MoveEECall,
+    Proposal,
+    SetGripperArgs,
+    SetGripperCall,
+)
+from nomad.core.codes import (
+    ActionStatus,
+    FeedbackCode,
+    ModelResponseStatus,
+    ObservationMode,
+    OutcomeStatus,
+    RunStatus,
+    StopReason,
+)
+from nomad.core.errors import ManifestError, MissingDependencyError, ModelError, NomadError
+from nomad.core.manifest import (
+    EmbodimentManifest,
+    load_manifest,
+    parse_manifest,
+    proposal_json_schema,
+)
+from nomad.core.model_io import AgentContext, HistoryEntry, ModelResponse, Usage
+from nomad.core.observations import CameraFrame, CameraIntrinsics, Observation, RobotState
+from nomad.core.protocols import DescribesRunConfig, Embodiment, ModelProvider, SupportsVideo
+from nomad.core.results import ActionResult, TaskOutcome
+from nomad.core.run import RunConfig, RunMetrics, RunResult
+from nomad.core.types import PROTOCOL_VERSION, Pose, Record
+
+__all__ = [
+    "PROTOCOL_VERSION",
+    "Action",
+    "ActionCall",
+    "ActionResult",
+    "ActionStatus",
+    "AgentContext",
+    "CameraFrame",
+    "CameraIntrinsics",
+    "DescribesRunConfig",
+    "Embodiment",
+    "EmbodimentManifest",
+    "FeedbackCode",
+    "HistoryEntry",
+    "ManifestError",
+    "MissingDependencyError",
+    "ModelError",
+    "ModelProvider",
+    "ModelResponse",
+    "ModelResponseStatus",
+    "MoveEEArgs",
+    "MoveEECall",
+    "NomadError",
+    "Observation",
+    "ObservationMode",
+    "OutcomeStatus",
+    "Pose",
+    "Proposal",
+    "Record",
+    "RobotState",
+    "RunConfig",
+    "RunMetrics",
+    "RunResult",
+    "RunStatus",
+    "SetGripperArgs",
+    "SetGripperCall",
+    "StopReason",
+    "SupportsVideo",
+    "TaskOutcome",
+    "Usage",
+    "load_manifest",
+    "parse_manifest",
+    "proposal_json_schema",
+]

nomad/core/actions.py

@@ -0,0 +1,88 @@
+"""Action vocabulary: what a model may propose and what the runtime dispatches.
+
+A model produces a :class:`Proposal` of :data:`ActionCall` items (name plus typed
+arguments). Only the runtime wraps a call into an :class:`Action`, attaching the
+execution identifiers and freshness deadline the model must not control.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated, Literal, TypeAlias
+
+from pydantic import Field, model_validator
+
+from nomad.core.types import (
+    FiniteFloat,
+    PositiveFloat,
+    Record,
+    UnitInterval,
+    Vec3,
+)
+
+
+class MoveEEArgs(Record):
+    """Total end-effector displacement in a declared frame, bounded in duration."""
+
+    frame: Annotated[str, Field(min_length=1)]
+    translation_m: Vec3
+    rotation_vector_rad: Vec3
+    duration_s: PositiveFloat
+
+
+class SetGripperArgs(Record):
+    """Desired closure: 0 is fully open, 1 is fully closed. Not a force command."""
+
+    closure: UnitInterval
+    duration_s: PositiveFloat
+
+
+class MoveEECall(Record):
+    name: Literal["move_ee"]
+    arguments: MoveEEArgs
+
+
+class SetGripperCall(Record):
+    name: Literal["set_gripper"]
+    arguments: SetGripperArgs
+
+
+ActionCall: TypeAlias = Annotated[MoveEECall | SetGripperCall, Field(discriminator="name")]
+
+
+class Proposal(Record):
+    """One model decision.
+
+    v0.1 rule: exactly one action unless ``request_finish`` is set, in which case
+    zero or one action is allowed. A finish request only triggers an independent
+    outcome check; it never marks the task successful by itself.
+    """
+
+    schema_version: Literal["0.1"]
+    actions: tuple[ActionCall, ...]
+    request_finish: bool
+    rationale: Annotated[str, Field(max_length=500)] | None = None
+
+    @model_validator(mode="after")
+    def _action_count(self) -> Proposal:
+        n = len(self.actions)
+        if self.request_finish and n > 1:
+            raise ValueError("a finish request may include at most one action")
+        if not self.request_finish and n != 1:
+            raise ValueError(f"exactly one action is required without request_finish (got {n})")
+        return self
+
+
+class Action(Record):
+    """Runtime execution envelope around one validated call."""
+
+    call: ActionCall
+    episode_id: str
+    action_id: str
+    observation_id: str
+    manifest_version: str
+    deadline_wall_s: FiniteFloat
+    protocol_version: Literal["0.1"] = "0.1"
+
+    @property
+    def name(self) -> str:
+        return self.call.name

nomad/core/codes.py

@@ -0,0 +1,103 @@
+"""Status and feedback vocabularies shared by the runtime, adapters, and traces.
+
+Every enum is a ``str`` enum so values serialize as plain strings in JSON
+traces and can be compared against literals in tests and analysis scripts.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+
+
+class _StrEnum(str, Enum):
+    def __str__(self) -> str:
+        return str(self.value)
+
+
+class ActionStatus(_StrEnum):
+    """What happened to one dispatched (or refused) action."""
+
+    COMPLETED = "completed"
+    REJECTED = "rejected"
+    FAILED = "failed"
+    TIMED_OUT = "timed_out"
+    CANCELLED = "cancelled"
+
+
+class FeedbackCode(_StrEnum):
+    """Machine-readable reason attached to every action result and rejection."""
+
+    OK = "ok"
+
+    # Structural problems with a model proposal.
+    MALFORMED_PROPOSAL = "malformed_proposal"
+    UNKNOWN_ACTION = "unknown_action"
+    UNKNOWN_FIELD = "unknown_field"
+    NONFINITE_VALUE = "nonfinite_value"
+    INVALID_DIMENSION = "invalid_dimension"
+    OUT_OF_RANGE = "out_of_range"
+
+    # Semantic problems relative to the active embodiment and observation.
+    UNSUPPORTED_ACTION = "unsupported_action"
+    UNSUPPORTED_FRAME = "unsupported_frame"
+    TRANSLATION_LIMIT = "translation_limit"
+    ROTATION_LIMIT = "rotation_limit"
+    DURATION_LIMIT = "duration_limit"
+    WORKSPACE_VIOLATION = "workspace_violation"
+    STALE_OBSERVATION = "stale_observation"
+    MANIFEST_MISMATCH = "manifest_mismatch"
+
+    # Execution outcomes reported by an embodiment.
+    TARGET_NOT_REACHED = "target_not_reached"
+    EXECUTION_INTERRUPTED = "execution_interrupted"
+
+    # Model-provider and loop-level feedback.
+    PROVIDER_ERROR = "provider_error"
+    PROVIDER_REFUSAL = "provider_refusal"
+    FINISH_NOT_CONFIRMED = "finish_not_confirmed"
+    CANCELLED = "cancelled"
+
+
+class OutcomeStatus(_StrEnum):
+    """Task outcome as judged by the embodiment's evaluator, never by the model."""
+
+    SUCCESS = "success"
+    FAILURE = "failure"
+    ONGOING = "ongoing"
+    UNKNOWN = "unknown"
+
+
+class ModelResponseStatus(_StrEnum):
+    OK = "ok"
+    ERROR = "error"
+    REFUSAL = "refusal"
+
+
+class ObservationMode(_StrEnum):
+    """Which state the model may see. Results from the two modes are reported separately."""
+
+    RGB_PROPRIO = "rgb_proprio"
+    PRIVILEGED_STATE = "privileged_state"
+
+
+class RunStatus(_StrEnum):
+    SUCCESS = "success"
+    FAILURE = "failure"
+    BUDGET_EXHAUSTED = "budget_exhausted"
+    CANCELLED = "cancelled"
+    ERROR = "error"
+
+
+class StopReason(_StrEnum):
+    """Why the runtime loop ended."""
+
+    TASK_SUCCESS = "task_success"
+    TASK_FAILURE = "task_failure"
+    MAX_DECISIONS = "max_decisions"
+    MAX_WALL_TIME = "max_wall_time"
+    MAX_SIM_TIME = "max_sim_time"
+    MAX_PROVIDER_CALLS = "max_provider_calls"
+    MAX_TOKENS = "max_tokens"
+    MAX_CONSECUTIVE_FAILURES = "max_consecutive_failures"
+    CANCELLED = "cancelled"
+    ERROR = "error"

nomad/core/errors.py

@@ -0,0 +1,19 @@
+"""Exception types. Bad model output is reported as feedback, not raised."""
+
+from __future__ import annotations
+
+
+class NomadError(Exception):
+    """Base class for errors raised by Nomad itself."""
+
+
+class ManifestError(NomadError):
+    """An embodiment manifest could not be loaded or is inconsistent."""
+
+
+class ModelError(NomadError):
+    """A model provider failed in a way its adapter could not normalize."""
+
+
+class MissingDependencyError(NomadError, ImportError):
+    """An optional integration was requested without its dependencies installed."""