sequence-base 0.3.0__tar.gz → 0.3.1__tar.gz

{sequence_base-0.3.0 → sequence_base-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: sequence-base
-Version: 0.3.0
+Version: 0.3.1
 Summary: Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness.
 Requires-Python: >=3.11
 Requires-Dist: pydantic>=2.0

{sequence_base-0.3.0 → sequence_base-0.3.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sequence-base"
-version = "0.3.0"
+version = "0.3.1"
 description = "Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness."
 requires-python = ">=3.11"
 dependencies = ["pydantic>=2.0"]

{sequence_base-0.3.0 → sequence_base-0.3.1}/sequence_base/__init__.py

@@ -23,6 +23,7 @@ from sequence_base.contract import (  # noqa: F401  (re-export surface)
     AdapterTransform,
     Autoscaler,
     BenchmarkRegisterRequest,
+    BenchmarkStatusResponse,
     BenchmarkSpec,
     BootstrapResponse,
     ClampInfo,
@@ -115,7 +116,7 @@ __all__ = [
     # contract 3 — connect / lease
     "Lease", "ConnectRequest", "ConnectResponse",
     # contract 5 — benchmark + eval
-    "Variant", "Conditions", "RenderSpec", "Step", "BenchmarkSpec", "BenchmarkRegisterRequest",
+    "Variant", "Conditions", "RenderSpec", "Step", "BenchmarkSpec", "BenchmarkRegisterRequest", "BenchmarkStatusResponse",
     "EvalRequest", "ClampInfo", "Estimate", "EvalResponse", "EvalDryRunResponse", "RolloutResult",
     "EvalStatus", "SuiteAgg", "TaskAgg", "MetricAgg", "LossyTransform", "EvalReport",
     "AdapterKind", "AdapterTransform",
@@ -131,4 +132,4 @@ __all__ = [
     "auth", "WorkerLogEvent",
 ]
 
-__version__ = "0.3.0"
+__version__ = "0.3.1"

{sequence_base-0.3.0 → sequence_base-0.3.1}/sequence_base/contract/__init__.py

@@ -50,6 +50,7 @@ from sequence_base.contract.connect import ConnectRequest, ConnectResponse, Leas
 # contract 5 — benchmark registration + eval run / report / adapter
 from sequence_base.contract.benchmarks import (
     BenchmarkRegisterRequest,
+    BenchmarkStatusResponse,
     BenchmarkSpec,
     Conditions,
     RenderSpec,
@@ -100,7 +101,7 @@ __all__ = [
     # contract 3
     "Lease", "ConnectRequest", "ConnectResponse",
     # contract 5
-    "Variant", "Conditions", "RenderSpec", "Step", "BenchmarkSpec", "BenchmarkRegisterRequest",
+    "Variant", "Conditions", "RenderSpec", "Step", "BenchmarkSpec", "BenchmarkRegisterRequest", "BenchmarkStatusResponse",
     "EvalRequest", "ClampInfo", "Estimate", "EvalResponse", "EvalDryRunResponse", "RolloutResult",
     "EvalStatus", "SuiteAgg", "TaskAgg", "MetricAgg", "LossyTransform", "EvalReport",
     "AdapterKind", "AdapterTransform",

{sequence_base-0.3.0 → sequence_base-0.3.1}/sequence_base/contract/benchmarks.py

@@ -15,8 +15,9 @@ from __future__ import annotations
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
 from sequence_base.contract.accounts import SecretRef
+from sequence_base.contract.deployments import CodeBundle, DeploymentState
 from sequence_base.contract.flags import BENCHMARK_BINDABLE, SeqFlags
-from sequence_base.contract.policy import ImageSpec, VolumeRef
+from sequence_base.contract.policy import ActionContract, ImageSpec, VolumeRef
 
 __all__ = [
     "Variant",
@@ -25,6 +26,7 @@ __all__ = [
     "Step",
     "BenchmarkSpec",
     "BenchmarkRegisterRequest",
+    "BenchmarkStatusResponse",
 ]
 
 
@@ -128,3 +130,23 @@ class BenchmarkRegisterRequest(BaseModel):
     render: RenderSpec | None = None
     cpu: int | None = None
     memory_gb: int | None = None
+    # The benchmark's own source (reset/check hooks, its adapters), shipped like a policy's — the eval worker
+    # imports handler_ref and any adapter ref from it. None only when the image installs the module itself.
+    code: CodeBundle | None = None
+    # Which suite each task belongs to (task -> suite), so reports aggregate per suite without the platform
+    # guessing from task names. A task not listed here has no suite.
+    suite_of: dict[str, str] = Field(default_factory=dict)
+    # The action contract the benchmark's sim consumes; eval compares it with the policy's and requires an
+    # adapter when they differ.
+    action_contract: ActionContract | None = None
+
+
+class BenchmarkStatusResponse(BaseModel):
+    """GET /v1/benchmarks/{name} — the build state of a registered benchmark (same lifecycle as a deployment:
+    building → ready | failed), with the failure reason when failed."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    name: str
+    status: DeploymentState
+    error: str | None = None

{sequence_base-0.3.0 → sequence_base-0.3.1}/sequence_base/contract/eval.py

@@ -177,6 +177,9 @@ class EvalReport(BaseModel):
     adapter_lossy: list[LossyTransform]
     videos: list[str] = Field(default_factory=list, description="mp4 pointers from the platform tool")
     incomplete: bool | None = None
+    errors: list[str] = Field(
+        default_factory=list, description="Why episodes or the run failed, so an incomplete report says why"
+    )
 
 
 class AdapterKind(str, Enum):

{sequence_base-0.3.0 → sequence_base-0.3.1}/sequence_base/schema_export.py

@@ -36,6 +36,7 @@ from sequence_base.contract.act import (
 )
 from sequence_base.contract.benchmarks import (
     BenchmarkRegisterRequest,
+    BenchmarkStatusResponse,
     BenchmarkSpec,
     Conditions,
     RenderSpec,
@@ -128,6 +129,7 @@ WIRE_MODELS: dict[str, type[BaseModel]] = {
     "Step": Step,
     "BenchmarkSpec": BenchmarkSpec,
     "BenchmarkRegisterRequest": BenchmarkRegisterRequest,
+    "BenchmarkStatusResponse": BenchmarkStatusResponse,
     # contract 5 — eval run / report / adapter
     "EvalRequest": EvalRequest,
     "ClampInfo": ClampInfo,

{sequence_base-0.3.0 → sequence_base-0.3.1}/sequence_base/schemas/BenchmarkRegisterRequest.json

@@ -1,5 +1,71 @@
 {
   "$defs": {
+    "ActionContract": {
+      "additionalProperties": false,
+      "description": "The fixed obs→ActionChunk contract a policy promises. Every field required; `action_space` reuses\ncontract 4's enum (not a free string). Each ActionChunk echoes exactly these invariants back.",
+      "properties": {
+        "action_dim": {
+          "title": "Action Dim",
+          "type": "integer"
+        },
+        "action_space": {
+          "$ref": "#/$defs/ActionSpace"
+        },
+        "control_frequency_hz": {
+          "title": "Control Frequency Hz",
+          "type": "number"
+        },
+        "robot": {
+          "title": "Robot",
+          "type": "string"
+        }
+      },
+      "required": [
+        "robot",
+        "action_space",
+        "action_dim",
+        "control_frequency_hz"
+      ],
+      "title": "ActionContract",
+      "type": "object"
+    },
+    "ActionSpace": {
+      "description": "Must be explicit: the same floats mean opposite things across spaces — executing an absolute\njoint position as a delta flings the arm into its limit stops. {schema.py:124}",
+      "enum": [
+        "joint_absolute",
+        "joint_delta",
+        "ee_absolute",
+        "ee_delta"
+      ],
+      "title": "ActionSpace",
+      "type": "string"
+    },
+    "CodeBundle": {
+      "additionalProperties": false,
+      "description": "The policy's own source, shipped with the deploy so the worker can import `policy.handler_ref`\n(what Modal's automount does for the entrypoint module). A gzip'd tar of the handler's top-level module\nor package, base64 on the wire, with the sha256 of the decoded bytes so a truncated or altered upload\nis refused at the edge instead of failing as an ImportError on a GPU.",
+      "properties": {
+        "data": {
+          "title": "Data",
+          "type": "string"
+        },
+        "format": {
+          "const": "tar.gz",
+          "default": "tar.gz",
+          "title": "Format",
+          "type": "string"
+        },
+        "sha256": {
+          "title": "Sha256",
+          "type": "string"
+        }
+      },
+      "required": [
+        "data",
+        "sha256"
+      ],
+      "title": "CodeBundle",
+      "type": "object"
+    },
     "Conditions": {
       "additionalProperties": false,
       "description": "The eval matrix — structured, not a bare dict. `tasks` and `seeds` required; `variants` optional.",
@@ -228,6 +294,17 @@
   "additionalProperties": false,
   "description": "POST /v1/benchmarks — register body, same types as the spec (ImageSpec / Conditions / VolumeRef /\nRenderSpec). Carries assets / render / cpu / memory_gb / max_steps so the gateway can build the image,\nprovision the `assets` Volume, and register in one call.",
   "properties": {
+    "action_contract": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/ActionContract"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null
+    },
     "assets": {
       "items": {
         "$ref": "#/$defs/VolumeRef"
@@ -235,6 +312,17 @@
       "title": "Assets",
       "type": "array"
     },
+    "code": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/CodeBundle"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null
+    },
     "conditions": {
       "$ref": "#/$defs/Conditions"
     },
@@ -287,6 +375,13 @@
         }
       ],
       "default": null
+    },
+    "suite_of": {
+      "additionalProperties": {
+        "type": "string"
+      },
+      "title": "Suite Of",
+      "type": "object"
     }
   },
   "required": [

sequence_base-0.3.1/sequence_base/schemas/BenchmarkStatusResponse.json

@@ -0,0 +1,44 @@
+{
+  "$defs": {
+    "DeploymentState": {
+      "description": "GET status — a restricted enum, not a free string (same discipline as EvalStatus).",
+      "enum": [
+        "building",
+        "ready",
+        "failed"
+      ],
+      "title": "DeploymentState",
+      "type": "string"
+    }
+  },
+  "additionalProperties": false,
+  "description": "GET /v1/benchmarks/{name} — the build state of a registered benchmark (same lifecycle as a deployment:\nbuilding → ready | failed), with the failure reason when failed.",
+  "properties": {
+    "error": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": null,
+      "title": "Error"
+    },
+    "name": {
+      "title": "Name",
+      "type": "string"
+    },
+    "status": {
+      "$ref": "#/$defs/DeploymentState"
+    }
+  },
+  "required": [
+    "name",
+    "status"
+  ],
+  "title": "BenchmarkStatusResponse",
+  "type": "object",
+  "x-contract-version": "0.4.1"
+}

{sequence_base-0.3.0 → sequence_base-0.3.1}/sequence_base/schemas/EvalReport.json

@@ -134,6 +134,14 @@
       "title": "Dashboard Url",
       "type": "string"
     },
+    "errors": {
+      "description": "Why episodes or the run failed, so an incomplete report says why",
+      "items": {
+        "type": "string"
+      },
+      "title": "Errors",
+      "type": "array"
+    },
     "incomplete": {
       "anyOf": [
         {

{sequence_base-0.3.0 → sequence_base-0.3.1}/tests/test_contracts.py

@@ -33,6 +33,7 @@ from sequence_base.contract.act import (
 )
 from sequence_base.contract.benchmarks import (
     BenchmarkRegisterRequest,
+    BenchmarkStatusResponse,
     BenchmarkSpec,
     Conditions,
     RenderSpec,
@@ -153,7 +154,10 @@ INSTANCES: dict[str, BaseModel] = {
                                    bindings={SeqFlags.SETUP: "setup", SeqFlags.RESET: "reset",
                                              SeqFlags.CHECK: "check", SeqFlags.SCORE: "score"}),
     "BenchmarkRegisterRequest": BenchmarkRegisterRequest(name="libero", image=_IMG, conditions=_COND,
-                                                         max_steps=400, handler_ref="m:B"),
+                                                         max_steps=400, handler_ref="m:B",
+                                                         code=CodeBundle.from_bytes(b"bench source"),
+                                                         suite_of={"pick": "spatial"}, action_contract=_AC),
+    "BenchmarkStatusResponse": BenchmarkStatusResponse(name="libero", status=DeploymentState.READY),
     "EvalRequest": EvalRequest(model="pi05", benchmark="libero"),
     "ClampInfo": ClampInfo(limit="account_quota", requested=8, granted=4),
     "Estimate": _EST,
@@ -444,7 +448,7 @@ def test_eval_report_metrics_passthrough_no_overall_sr():
     assert "success_rate" not in EvalReport.model_fields
     assert EvalReport.model_fields["metrics"].annotation == dict[str, MetricAgg]
     assert set(EvalReport.model_fields) == {
-        "status", "metrics", "cost_usd", "dashboard_url", "adapter_lossy", "videos", "incomplete"}
+        "status", "metrics", "cost_usd", "dashboard_url", "adapter_lossy", "videos", "incomplete", "errors"}
     rep = EvalReport(status=EvalStatus.COMPLETED,
                      metrics={"success": MetricAgg(overall=0.5, n=4),
                               "time_to_grasp": MetricAgg(overall=2.1, n=4)},