sequence-base 0.3.18__tar.gz → 0.3.21__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. {sequence_base-0.3.18 → sequence_base-0.3.21}/PKG-INFO +1 -1
  2. {sequence_base-0.3.18 → sequence_base-0.3.21}/pyproject.toml +1 -1
  3. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/constants/version.py +4 -1
  4. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/contract/accounts.py +2 -1
  5. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/contract/benchmarks.py +14 -4
  6. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/contract/deployments.py +19 -3
  7. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/contract/eval.py +67 -1
  8. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/contract/flags.py +10 -3
  9. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/contract/policy.py +7 -4
  10. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ActRequest.json +1 -1
  11. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ActResponse.json +1 -1
  12. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ActionChunk.json +1 -1
  13. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ActionContract.json +1 -1
  14. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ActionStep.json +1 -1
  15. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/AdapterTransform.json +1 -1
  16. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/AddLocal.json +1 -1
  17. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/Autoscaler.json +1 -1
  18. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/BenchmarkConditionSpec.json +1 -1
  19. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/BenchmarkRegisterRequest.json +20 -1
  20. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/BenchmarkSpec.json +25 -5
  21. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/BenchmarkStatusResponse.json +1 -1
  22. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/BootstrapResponse.json +11 -4
  23. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/Camera.json +1 -1
  24. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ClampInfo.json +1 -1
  25. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/CodeBundle.json +1 -1
  26. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/Concurrency.json +1 -1
  27. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/Conditions.json +1 -1
  28. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ConnectRequest.json +1 -1
  29. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ConnectResponse.json +1 -1
  30. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/DeploymentAccepted.json +1 -1
  31. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/DeploymentPolicy.json +17 -1
  32. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/DeploymentRequest.json +27 -4
  33. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/DeploymentStatusResponse.json +1 -1
  34. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/Embodiment.json +1 -1
  35. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ErrorBody.json +1 -1
  36. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ErrorCodes.json +1 -1
  37. sequence_base-0.3.21/sequence_base/schemas/Estimate.json +52 -0
  38. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/EvalDryRunResponse.json +48 -1
  39. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/EvalReport.json +215 -2
  40. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/EvalRequest.json +40 -2
  41. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/EvalResponse.json +40 -1
  42. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ImageFrame.json +1 -1
  43. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ImageSpec.json +1 -1
  44. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ImageStepAddLocal.json +1 -1
  45. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ImageStepApt.json +1 -1
  46. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ImageStepEnv.json +1 -1
  47. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ImageStepRun.json +1 -1
  48. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ImageStepUvPip.json +1 -1
  49. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/InferParam.json +1 -1
  50. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/Lease.json +1 -1
  51. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/LossyTransform.json +1 -1
  52. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/MetricAgg.json +1 -1
  53. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/Observation.json +1 -1
  54. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/ObservationContract.json +1 -1
  55. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/PolicyRegistration.json +22 -9
  56. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/PolicySpec.json +13 -6
  57. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/Proprioception.json +1 -1
  58. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/RenderSpec.json +1 -1
  59. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/Resources.json +1 -1
  60. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/RolloutResult.json +1 -1
  61. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/SecretCreateRequest.json +1 -1
  62. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/SecretInfo.json +1 -1
  63. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/SecretRef.json +1 -1
  64. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/SeqFlags.json +6 -2
  65. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/Step.json +1 -1
  66. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/SuiteAgg.json +1 -1
  67. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/TaskAgg.json +1 -1
  68. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/Timeouts.json +1 -1
  69. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/UploadTarget.json +1 -1
  70. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/UsageReport.json +1 -1
  71. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/Variant.json +1 -1
  72. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/VolumeAttach.json +1 -1
  73. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/VolumeFile.json +1 -1
  74. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/VolumeInfo.json +1 -1
  75. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/VolumeMount.json +1 -1
  76. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/VolumeRef.json +1 -1
  77. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/VolumeUploadRequest.json +1 -1
  78. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/VolumeUploadResponse.json +1 -1
  79. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schemas/WarmingError.json +1 -1
  80. sequence_base-0.3.18/sequence_base/schemas/Estimate.json +0 -25
  81. {sequence_base-0.3.18 → sequence_base-0.3.21}/.gitignore +0 -0
  82. {sequence_base-0.3.18 → sequence_base-0.3.21}/README.md +0 -0
  83. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/__init__.py +0 -0
  84. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/auth.py +0 -0
  85. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/codec/__init__.py +0 -0
  86. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/codec/image.py +0 -0
  87. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/constants/__init__.py +0 -0
  88. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/constants/env.py +0 -0
  89. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/constants/headers.py +0 -0
  90. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/contract/__init__.py +0 -0
  91. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/contract/act.py +0 -0
  92. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/contract/connect.py +0 -0
  93. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/contract/errors.py +0 -0
  94. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/contract/io.py +0 -0
  95. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/contract/templates.py +0 -0
  96. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/contract/volumes.py +0 -0
  97. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/contract/wire.py +0 -0
  98. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/errors/__init__.py +0 -0
  99. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/errors/transport.py +0 -0
  100. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/obs/__init__.py +0 -0
  101. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/obs/log.py +0 -0
  102. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/obs/rid.py +0 -0
  103. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/presets/__init__.py +0 -0
  104. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/presets/robot_arm.py +0 -0
  105. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/schema_export.py +0 -0
  106. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/telemetry/__init__.py +0 -0
  107. {sequence_base-0.3.18 → sequence_base-0.3.21}/sequence_base/telemetry/events.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: sequence-base
3
- Version: 0.3.18
3
+ Version: 0.3.21
4
4
  Summary: Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness.
5
5
  Requires-Python: >=3.10
6
6
  Requires-Dist: pydantic>=2.0
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "sequence-base"
3
- version = "0.3.18"
3
+ version = "0.3.21"
4
4
  description = "Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness."
5
5
  # 3.10, not 3.11: `seq` (which imports this) must run on ROS 2 Humble's Python 3.10, and nothing here needs 3.11.
6
6
  # {2026-10-03 the full suite on CPython 3.10.20: "147 PASSED"; vermin --eval-annotations: "MINIMUM REQUIRED VERSIONS: 3.9"}
@@ -4,7 +4,10 @@
4
4
  # across deployments / repos can tell which version of a shape it is validating against. MUST change
5
5
  # whenever any shared wire shape changes (a field added / removed / retyped) — that is what makes
6
6
  # contract evolution mechanically detectable instead of eyeballed. Bump on any shape change.
7
- CONTRACT_VERSION = "0.4.15" # 0.4.14 -> 0.4.15: limits count GPUs and CPU/memory are billed like Modal — Resources.cpu / memory_gb optional floats + cpu_limit / memory_limit_gb; Autoscaler.max_gpus / on_full / queue_timeout_s replace max_containers; Concurrency fields optional + replicas_per_gpu / gpus_per_replica; DeploymentStatusResponse.reason; EvalRequest.policy_instances / policy_gpu / benchmark_instances / benchmark_gpu / envs replace gpus / envs_per_gpu; EvalStatus.QUEUED; EvalCards / EvalQueue
7
+ CONTRACT_VERSION = "0.4.18" # 0.4.17 -> 0.4.18: DeploymentPolicy.models — the platform models a policy calls while it runs (catalogue ids), which the act key driving it may call on /v1/chat and /v1/decide (gs-plans models-api/11)
8
+ # previous: 0.4.17 # 0.4.16 -> 0.4.17: weights optional — PolicySpec.weights, DeploymentRequest.weights and BootstrapResponse.weights_uri may be None, for a policy without a checkpoint (nothing copied or mounted; gs-plans storage T13)
9
+ # previous: 0.4.16 # 0.4.15 -> 0.4.16: batched benchmarks — SeqFlags.OBSERVE / STEP, BenchmarkSpec / BenchmarkRegisterRequest.envs and .batched; per-arm eval reports — EvalReport.arms / paired / timing / stop_reason, EvalStatus.CANCELED, Estimate.breakdown / default_timing, EvalResponse.arms, EvalDryRunResponse.layout / notes
10
+ # previous: 0.4.15 # 0.4.14 -> 0.4.15: limits count GPUs and CPU/memory are billed like Modal — Resources.cpu / memory_gb optional floats + cpu_limit / memory_limit_gb; Autoscaler.max_gpus / on_full / queue_timeout_s replace max_containers; Concurrency fields optional + replicas_per_gpu / gpus_per_replica; DeploymentStatusResponse.reason; EvalRequest.policy_instances / policy_gpu / benchmark_instances / benchmark_gpu / envs replace gpus / envs_per_gpu; EvalStatus.QUEUED; EvalCards / EvalQueue
8
11
  # previous: 0.4.14 # 0.4.13 -> 0.4.14: the /act wire is opaque (ActRequest.inputs / ActResponse.outputs); action / observation contracts optional + DeploymentPolicy/PolicySpec/ConnectResponse .io; the robot-arm types live in sequence_base.presets.robot_arm (contract.act / contract.io re-export them); ImageStepAddLocal (seq.Image.add_local_*); ActionSpace.JOINT_VELOCITY (pi0.5 DROID's joint velocities + gripper position)
9
12
  # previous: 0.4.13 # 0.4.12 -> 0.4.13: template://<name>[@v<n>] — official template weights (parse_template); gs:// / s3:// / https:// still accepted until their removal
10
13
  # previous: 0.4.12 # 0.4.11 -> 0.4.12: one error definition — ErrorBody is every non-2xx body (detail, code, retryable, retry_after_s, request_id, hint, details), ErrorCode covers every case, ERRORS / ErrorCodes.json give each code its status, retryability, SDK class and CLI exit code
@@ -76,7 +76,8 @@ class BootstrapResponse(BaseModel):
76
76
  tls_key: str = Field(..., description="Self-signed TLS private key (down-direction platform secret)")
77
77
  tls_cert: str = Field(..., description="Self-signed TLS certificate (public)")
78
78
  env: dict[str, str] = Field(..., description="Policy's user secrets, expanded ENV_KEY → value")
79
- weights_uri: str
79
+ # None: a policy without a checkpoint — nothing was copied or mounted. {gs-plans/storage T13}
80
+ weights_uri: str | None = None
80
81
  lease: Lease
81
82
 
82
83
 
@@ -16,7 +16,7 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator
16
16
 
17
17
  from sequence_base.contract.accounts import SecretRef
18
18
  from sequence_base.contract.deployments import CodeBundle, DeploymentState
19
- from sequence_base.contract.flags import BENCHMARK_BINDABLE, SeqFlags
19
+ from sequence_base.contract.flags import BENCHMARK_BATCHED, BENCHMARK_BINDABLE, SeqFlags
20
20
  from sequence_base.contract.policy import ActionContract, ImageSpec, VolumeRef
21
21
 
22
22
  __all__ = [
@@ -92,8 +92,10 @@ class BenchmarkSpec(BaseModel):
92
92
  conditions: Conditions
93
93
  max_steps: int = Field(..., description="Platform safety cap; truncates and records truncated")
94
94
  handler_ref: str
95
- bindings: dict[SeqFlags, str] = Field(..., description="SETUP / RESET / CHECK / SCORE → method name")
95
+ bindings: dict[SeqFlags, str] = Field(..., description="SETUP / RESET / CHECK / SCORE (+ OBSERVE / STEP) → method name")
96
96
  gpu: str | list[str] | None = None
97
+ # How many environments one benchmark instance runs at once when an eval does not say (the author's default).
98
+ envs: int | None = Field(default=None, ge=1, le=65536)
97
99
  assets: list[VolumeRef] = Field(default_factory=list)
98
100
  render: RenderSpec | None = None
99
101
  cpu: int | None = None
@@ -108,9 +110,12 @@ class BenchmarkSpec(BaseModel):
108
110
  if bad:
109
111
  names = ", ".join(sorted(f.name or str(f.value) for f in bad))
110
112
  raise ValueError(
111
- f"benchmark bindings may only cover SETUP / RESET / CHECK / SCORE; got disallowed {names}"
113
+ f"benchmark bindings may only cover SETUP / RESET / CHECK / SCORE / OBSERVE / STEP; got disallowed {names}"
112
114
  " (ROLLOUT is deprecated and must not be bound)"
113
115
  )
116
+ batched = set(self.bindings) & set(BENCHMARK_BATCHED)
117
+ if batched and batched != set(BENCHMARK_BATCHED):
118
+ raise ValueError("a batched benchmark binds both OBSERVE and STEP")
114
119
  return self
115
120
 
116
121
 
@@ -166,8 +171,13 @@ class BenchmarkRegisterRequest(BaseModel):
166
171
  timeout_s: float | None = Field(default=None, gt=0)
167
172
  # The adapter (module:Class in the benchmark's own code) an eval uses when it names none.
168
173
  default_adapter_ref: str | None = None
169
- # A GPU for each sim sandbox (a renderer or a GPU physics engine): a card name the platform offers. None: CPU.
174
+ # The card each benchmark instance runs on (L4 or RTX-PRO-6000); None takes the platform's default (L4). The
175
+ # simulation never runs on a CPU. {gs-plans/benchmark-internals/design.md "一、规则"}
170
176
  gpu: str | None = Field(default=None, max_length=32)
177
+ # How many environments one instance runs at once when an eval does not say; None takes 1.
178
+ envs: int | None = Field(default=None, ge=1, le=65536)
179
+ # Whether the benchmark is batched (binds OBSERVE and STEP and steps many environments in one call).
180
+ batched: bool = False
171
181
  # The Dockerfile's build context, as for a deployment.
172
182
  image_context: CodeBundle | None = None
173
183
 
@@ -157,12 +157,23 @@ class DeploymentPolicy(BaseModel):
157
157
  # The inference parameters @seq.infer takes beyond the observation (name -> type, default). None: an older
158
158
  # SDK that did not look; {}: the policy declares none.
159
159
  params: dict[str, InferParam] | None = Field(default=None, max_length=32)
160
+ # The platform models the policy calls while it runs (`@seq.policy(models=[...])`), by full catalogue id
161
+ # (accounts/sequences/models/<name>): the act key that drives this deployment may call exactly these on /v1/chat
162
+ # and /v1/decide, and nothing else it could spend on. None: the policy calls none (and an older SDK never sent it).
163
+ # {gs-plans models-api/11, user 2026-10-04 "act key should have ability to call any model involved in the policy"}
164
+ models: list[str] | None = Field(default=None, max_length=16)
160
165
 
161
166
  @model_validator(mode="after")
162
167
  def _param_names(self) -> "DeploymentPolicy":
163
168
  for name in self.params or {}:
164
169
  if not name.isidentifier() or name.startswith("_"):
165
170
  raise ValueError(f"inference parameter {name!r} is not a public Python identifier")
171
+ # A declared model is a catalogue id: no blanks, no duplicates (the gateway checks each one exists).
172
+ for model in self.models or []:
173
+ if not model or model != model.strip() or "/" not in model:
174
+ raise ValueError(f"policy model {model!r} is not a catalogue id (accounts/sequences/models/<name>)")
175
+ if self.models and len(set(self.models)) != len(self.models):
176
+ raise ValueError("policy models lists a model twice")
166
177
  return self
167
178
 
168
179
 
@@ -206,14 +217,15 @@ class CodeBundle(BaseModel):
206
217
 
207
218
 
208
219
  class DeploymentRequest(BaseModel):
209
- """The full `POST /v1/deployments` body. Required: name, image, weights, resources, policy (goal 18).
210
- autoscaler / concurrency / timeouts are typed optional knobs with sane defaults."""
220
+ """The full `POST /v1/deployments` body. Required: name, image, resources, policy (goal 18); weights unless the
221
+ policy has no checkpoint. autoscaler / concurrency / timeouts are typed optional knobs with sane defaults."""
211
222
 
212
223
  model_config = ConfigDict(extra="forbid")
213
224
 
214
225
  name: str
215
226
  image: ImageSpec
216
- weights: VolumeMount
227
+ # None: a policy without a checkpoint — no weights volume is made, copied or mounted. {gs-plans/storage T13}
228
+ weights: VolumeMount | None = None
217
229
  resources: Resources
218
230
  policy: DeploymentPolicy
219
231
  # The handler's source. None only when `handler_ref` is importable from the image itself (a package the
@@ -252,6 +264,10 @@ class DeploymentRequest(BaseModel):
252
264
  and, until they are removed, a gs:// or s3:// prefix or an https:// file — never empty. PolicySpec already enforces this for SDK-authored deploys; this holds it for any POSTed body."""
253
265
  from sequence_base.contract.policy import check_weights_parts
254
266
 
267
+ if self.weights is None:
268
+ if self.weights_name or self.extra_weights:
269
+ raise ValueError("weights parts need weights: name the first part's origin too")
270
+ return self
255
271
  check_weights_parts(self.weights_name, dict(self.extra_weights or {}))
256
272
  for what, ref in [("weights.uri", self.weights)] + [(f"extra_weights[{n!r}].uri", r)
257
273
  for n, r in (self.extra_weights or {}).items()]:
@@ -17,6 +17,9 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator
17
17
  __all__ = [
18
18
  "EvalRequest",
19
19
  "ClampInfo",
20
+ "ArmReport",
21
+ "PairedDiff",
22
+ "EvalTiming",
20
23
  "Estimate",
21
24
  "EvalResponse",
22
25
  "EvalDryRunResponse",
@@ -57,7 +60,10 @@ class EvalRequest(BaseModel):
57
60
  envs: int | None = Field(default=None, ge=1, le=100000)
58
61
  vlm_max_concurrency: int | None = None
59
62
  adapter: str | None = None
60
- compare: str | None = Field(default=None, description="Another model id to compare against")
63
+ # Arms run side by side in one eval: other models on the same episodes (each its own policy instances), and/or a
64
+ # grid of inference-parameter values (every combination is one arm).
65
+ compare: str | list[str] | None = Field(default=None, description="Other model ids to compare against")
66
+ sweep: dict[str, list[int | float | bool | str]] | None = Field(default=None, max_length=8)
61
67
  dry_run: bool = False
62
68
  # Only these tasks (of the selected suite / trials). {gs-plans/customization/benchmark.md "子集"}
63
69
  tasks: list[str] | None = Field(default=None, max_length=1000)
@@ -90,6 +96,10 @@ class Estimate(BaseModel):
90
96
  wall_clock_s: float
91
97
  cost_usd: float
92
98
  vlm_calls: int
99
+ # The cost split by machine and by resource (GPU / CPU / memory), when the gateway itemizes it.
100
+ breakdown: dict[str, float] | None = None
101
+ # True when no run of this benchmark version has been measured yet and the estimate uses the default step time.
102
+ default_timing: bool | None = None
93
103
 
94
104
 
95
105
  class EvalResponse(BaseModel):
@@ -103,6 +113,8 @@ class EvalResponse(BaseModel):
103
113
  clamped_by: ClampInfo | None = None
104
114
  # Set when the account was short of GPUs: the eval waits (up to its timeout) instead of being refused.
105
115
  queue: EvalQueue | None = None
116
+ # How many arms (model x swept parameters) the eval runs side by side.
117
+ arms: int | None = None
106
118
 
107
119
 
108
120
  class CardSide(BaseModel):
@@ -160,6 +172,10 @@ class EvalDryRunResponse(BaseModel):
160
172
 
161
173
  estimate: Estimate
162
174
  cards: EvalCards | None = None
175
+ # The eval's layout as it would run: per side the instances, card and environments (the user's numbers or the
176
+ # fixed defaults), and the arms. Advice only ever comes as `notes`; the platform never changes the numbers.
177
+ layout: dict | None = None
178
+ notes: list[str] = Field(default_factory=list)
163
179
 
164
180
 
165
181
  class RolloutResult(BaseModel):
@@ -190,6 +206,7 @@ class EvalStatus(str, Enum):
190
206
  RUNNING = "running"
191
207
  COMPLETED = "completed"
192
208
  FAILED = "failed"
209
+ CANCELED = "canceled"
193
210
 
194
211
 
195
212
  class SuiteAgg(BaseModel):
@@ -236,6 +253,49 @@ class LossyTransform(BaseModel):
236
253
  reason: str
237
254
 
238
255
 
256
+ class ArmReport(BaseModel):
257
+ """One arm of an eval (a model x one combination of swept inference parameters): its own metrics and episode
258
+ counts, so arms are compared side by side and never pooled."""
259
+
260
+ model_config = ConfigDict(extra="forbid")
261
+
262
+ arm: int
263
+ model: str
264
+ params: dict[str, int | float | bool | str] = Field(default_factory=dict)
265
+ metrics: dict[str, MetricAgg] = Field(default_factory=dict)
266
+ # Each metric's 95% confidence interval as [low, high].
267
+ intervals: dict[str, list[float]] = Field(default_factory=dict)
268
+ episodes: int = 0
269
+ failed: int = 0
270
+ not_run: int = 0
271
+ budget_stopped: bool = False
272
+
273
+
274
+ class PairedDiff(BaseModel):
275
+ """`compare`: one metric's difference between two arms over the seeds both ran (b - a), with its interval."""
276
+
277
+ model_config = ConfigDict(extra="forbid")
278
+
279
+ metric: str
280
+ arm_a: int
281
+ arm_b: int
282
+ n: int
283
+ mean_diff: float
284
+ interval: list[float]
285
+
286
+
287
+ class EvalTiming(BaseModel):
288
+ """Where an eval's time went: cold starts, and per step the simulation, rendering, the policy's batched forward and
289
+ the network around it, plus the forward's average batch size."""
290
+
291
+ model_config = ConfigDict(extra="forbid")
292
+
293
+ cold_start_s: dict[str, float] = Field(default_factory=dict)
294
+ step_ms: dict[str, float] = Field(default_factory=dict)
295
+ batch_size_mean: float | None = None
296
+ steps: int = 0
297
+
298
+
239
299
  class EvalReport(BaseModel):
240
300
  """GET /v1/eval/{id} (eval-raw revision). NO hard-coded `overall_sr` / `success_rate` top-level field —
241
301
  `metrics` is a passthrough dict keyed by the user's metric names (from `@seq.check`'s `Step.metrics`),
@@ -257,6 +317,12 @@ class EvalReport(BaseModel):
257
317
  errors: list[str] = Field(
258
318
  default_factory=list, description="Why episodes or the run failed, so an incomplete report says why"
259
319
  )
320
+ # Per arm (one entry for an eval with one arm), paired comparisons for `compare`, the timing breakdown, and why
321
+ # the run stopped early (canceled / budget), when it did.
322
+ arms: list[ArmReport] | None = None
323
+ paired: list[PairedDiff] | None = None
324
+ timing: EvalTiming | None = None
325
+ stop_reason: str | None = None
260
326
 
261
327
 
262
328
  class AdapterKind(str, Enum):
@@ -41,18 +41,25 @@ class SeqFlags(IntFlag):
41
41
  SCORE = 16384 # cross-episode aggregate (optional; default = mean of each numeric metric)
42
42
  RESET = 32768 # build one episode's env (gym-style handle) — eval-raw new hook, bit 15
43
43
  CHECK = 65536 # per step: read raw env state → done + user-named metrics — eval-raw new hook, bit 16
44
+ # A batched benchmark (a GPU sim that steps many environments at once) also binds these two; its RESET then
45
+ # takes (slots, conditions) and its CHECK (slots). {gs-plans/benchmark-internals/design.md "三、两种写法"}
46
+ OBSERVE = 131072 # read the observation of the given slots, one per slot — bit 17
47
+ STEP = 262144 # apply one action to each given slot — bit 18
44
48
  # DEPRECATED — the old single-hook benchmark model. Bit 8192 stays reserved (never reused for another
45
49
  # member) but is NOT bound in any registration; setup/reset/check/score replaces it. Kept only so the
46
50
  # value cannot be recycled and an old serialized 8192 still decodes to a known name.
47
51
  ROLLOUT = 8192
48
52
 
49
53
 
50
- # The benchmark lifecycle bits a live registration may bind (eval-raw): SETUP / RESET / CHECK / SCORE.
51
- # ROLLOUT is deliberately absent — deprecated, never bound.
54
+ # The benchmark lifecycle bits a live registration may bind (eval-raw): SETUP / RESET / CHECK / SCORE, and OBSERVE /
55
+ # STEP for a batched benchmark. ROLLOUT is deliberately absent — deprecated, never bound.
52
56
  BENCHMARK_BINDABLE: frozenset[SeqFlags] = frozenset(
53
- {SeqFlags.SETUP, SeqFlags.RESET, SeqFlags.CHECK, SeqFlags.SCORE}
57
+ {SeqFlags.SETUP, SeqFlags.RESET, SeqFlags.CHECK, SeqFlags.SCORE, SeqFlags.OBSERVE, SeqFlags.STEP}
54
58
  )
55
59
 
60
+ # The two bits that make a benchmark batched: present together or not at all.
61
+ BENCHMARK_BATCHED: frozenset[SeqFlags] = frozenset({SeqFlags.OBSERVE, SeqFlags.STEP})
62
+
56
63
  # Deprecated benchmark bits: reserved value, must not appear in any bindings.
57
64
  DEPRECATED_FLAGS: frozenset[SeqFlags] = frozenset({SeqFlags.ROLLOUT})
58
65
 
@@ -185,8 +185,9 @@ class PolicySpec(BaseModel):
185
185
  name: str
186
186
  gpu: str | list[str] = Field(..., description="A GPU type, or a list = cross-provider fallback")
187
187
  image: ImageSpec
188
- weights: str = Field(..., description="Weights URI: template:// (an official template), hf:// or vol:// (an "
189
- "uploaded volume)")
188
+ # None: the policy has no checkpoint (a closed-form controller); nothing is copied or mounted for it.
189
+ weights: str | None = Field(None, description="Weights URI: template:// (an official template), hf:// or vol:// "
190
+ "(an uploaded volume); none for a policy without a checkpoint")
190
191
  volumes: dict[str, VolumeRef] = Field(default_factory=dict)
191
192
  secrets: list[SecretRef] = Field(default_factory=list)
192
193
  min_containers: int = 0
@@ -217,11 +218,13 @@ class PolicySpec(BaseModel):
217
218
 
218
219
  @field_validator("weights")
219
220
  @classmethod
220
- def _weights_scheme(cls, v: str) -> str:
221
- return check_weights_uri(v, "weights")
221
+ def _weights_scheme(cls, v: str | None) -> str | None:
222
+ return None if v is None else check_weights_uri(v, "weights")
222
223
 
223
224
  @model_validator(mode="after")
224
225
  def _weights_parts(self) -> "PolicySpec":
226
+ if self.weights is None and (self.weights_name or self.extra_weights):
227
+ raise ValueError("weights parts need weights: name the first part's origin too")
225
228
  check_weights_parts(self.weights_name, {k: v for k, v in (self.extra_weights or {}).items()})
226
229
  for name, uri in (self.extra_weights or {}).items():
227
230
  check_weights_uri(uri, f"weights[{name!r}]")
@@ -65,5 +65,5 @@
65
65
  ],
66
66
  "title": "ActRequest",
67
67
  "type": "object",
68
- "x-contract-version": "0.4.15"
68
+ "x-contract-version": "0.4.18"
69
69
  }
@@ -75,5 +75,5 @@
75
75
  ],
76
76
  "title": "ActResponse",
77
77
  "type": "object",
78
- "x-contract-version": "0.4.15"
78
+ "x-contract-version": "0.4.18"
79
79
  }
@@ -99,5 +99,5 @@
99
99
  ],
100
100
  "title": "ActionChunk",
101
101
  "type": "object",
102
- "x-contract-version": "0.4.15"
102
+ "x-contract-version": "0.4.18"
103
103
  }
@@ -100,5 +100,5 @@
100
100
  ],
101
101
  "title": "ActionContract",
102
102
  "type": "object",
103
- "x-contract-version": "0.4.15"
103
+ "x-contract-version": "0.4.18"
104
104
  }
@@ -19,5 +19,5 @@
19
19
  ],
20
20
  "title": "ActionStep",
21
21
  "type": "object",
22
- "x-contract-version": "0.4.15"
22
+ "x-contract-version": "0.4.18"
23
23
  }
@@ -38,5 +38,5 @@
38
38
  ],
39
39
  "title": "AdapterTransform",
40
40
  "type": "object",
41
- "x-contract-version": "0.4.15"
41
+ "x-contract-version": "0.4.18"
42
42
  }
@@ -23,5 +23,5 @@
23
23
  ],
24
24
  "title": "AddLocal",
25
25
  "type": "object",
26
- "x-contract-version": "0.4.15"
26
+ "x-contract-version": "0.4.18"
27
27
  }
@@ -69,5 +69,5 @@
69
69
  },
70
70
  "title": "Autoscaler",
71
71
  "type": "object",
72
- "x-contract-version": "0.4.15"
72
+ "x-contract-version": "0.4.18"
73
73
  }
@@ -62,5 +62,5 @@
62
62
  ],
63
63
  "title": "BenchmarkConditionSpec",
64
64
  "type": "object",
65
- "x-contract-version": "0.4.15"
65
+ "x-contract-version": "0.4.18"
66
66
  }
@@ -523,6 +523,11 @@
523
523
  "title": "Assets",
524
524
  "type": "array"
525
525
  },
526
+ "batched": {
527
+ "default": false,
528
+ "title": "Batched",
529
+ "type": "boolean"
530
+ },
526
531
  "code": {
527
532
  "anyOf": [
528
533
  {
@@ -577,6 +582,20 @@
577
582
  "default": null,
578
583
  "title": "Default Adapter Ref"
579
584
  },
585
+ "envs": {
586
+ "anyOf": [
587
+ {
588
+ "maximum": 65536,
589
+ "minimum": 1,
590
+ "type": "integer"
591
+ },
592
+ {
593
+ "type": "null"
594
+ }
595
+ ],
596
+ "default": null,
597
+ "title": "Envs"
598
+ },
580
599
  "gpu": {
581
600
  "anyOf": [
582
601
  {
@@ -681,5 +700,5 @@
681
700
  ],
682
701
  "title": "BenchmarkRegisterRequest",
683
702
  "type": "object",
684
- "x-contract-version": "0.4.15"
703
+ "x-contract-version": "0.4.18"
685
704
  }
@@ -273,15 +273,18 @@
273
273
  8192,
274
274
  16384,
275
275
  32768,
276
- 65536
276
+ 65536,
277
+ 131072,
278
+ 262144
277
279
  ],
278
280
  "title": "SeqFlags",
279
281
  "type": "integer",
280
- "x-contract-version": "0.4.15",
282
+ "x-contract-version": "0.4.18",
281
283
  "x-flag-names": {
282
284
  "CHECK": 65536,
283
285
  "INFER": 16,
284
286
  "LOAD": 2,
287
+ "OBSERVE": 131072,
285
288
  "ON_EPISODE": 1024,
286
289
  "PLAN": 4,
287
290
  "RESET": 32768,
@@ -289,6 +292,7 @@
289
292
  "SCORE": 16384,
290
293
  "SETUP": 4096,
291
294
  "SHUTDOWN": 8,
295
+ "STEP": 262144,
292
296
  "WARMUP": 1
293
297
  }
294
298
  },
@@ -357,7 +361,7 @@
357
361
  "additionalProperties": {
358
362
  "type": "string"
359
363
  },
360
- "description": "SETUP / RESET / CHECK / SCORE → method name",
364
+ "description": "SETUP / RESET / CHECK / SCORE (+ OBSERVE / STEP) → method name",
361
365
  "propertyNames": {
362
366
  "enum": [
363
367
  "1",
@@ -370,7 +374,9 @@
370
374
  "8192",
371
375
  "16384",
372
376
  "32768",
373
- "65536"
377
+ "65536",
378
+ "131072",
379
+ "262144"
374
380
  ],
375
381
  "type": "string"
376
382
  },
@@ -392,6 +398,20 @@
392
398
  "default": null,
393
399
  "title": "Cpu"
394
400
  },
401
+ "envs": {
402
+ "anyOf": [
403
+ {
404
+ "maximum": 65536,
405
+ "minimum": 1,
406
+ "type": "integer"
407
+ },
408
+ {
409
+ "type": "null"
410
+ }
411
+ ],
412
+ "default": null,
413
+ "title": "Envs"
414
+ },
395
415
  "gpu": {
396
416
  "anyOf": [
397
417
  {
@@ -467,5 +487,5 @@
467
487
  ],
468
488
  "title": "BenchmarkSpec",
469
489
  "type": "object",
470
- "x-contract-version": "0.4.15"
490
+ "x-contract-version": "0.4.18"
471
491
  }
@@ -40,5 +40,5 @@
40
40
  ],
41
41
  "title": "BenchmarkStatusResponse",
42
42
  "type": "object",
43
- "x-contract-version": "0.4.15"
43
+ "x-contract-version": "0.4.18"
44
44
  }
@@ -58,8 +58,16 @@
58
58
  "type": "string"
59
59
  },
60
60
  "weights_uri": {
61
- "title": "Weights Uri",
62
- "type": "string"
61
+ "anyOf": [
62
+ {
63
+ "type": "string"
64
+ },
65
+ {
66
+ "type": "null"
67
+ }
68
+ ],
69
+ "default": null,
70
+ "title": "Weights Uri"
63
71
  }
64
72
  },
65
73
  "required": [
@@ -67,10 +75,9 @@
67
75
  "tls_key",
68
76
  "tls_cert",
69
77
  "env",
70
- "weights_uri",
71
78
  "lease"
72
79
  ],
73
80
  "title": "BootstrapResponse",
74
81
  "type": "object",
75
- "x-contract-version": "0.4.15"
82
+ "x-contract-version": "0.4.18"
76
83
  }
@@ -34,5 +34,5 @@
34
34
  ],
35
35
  "title": "Camera",
36
36
  "type": "object",
37
- "x-contract-version": "0.4.15"
37
+ "x-contract-version": "0.4.18"
38
38
  }
@@ -23,5 +23,5 @@
23
23
  ],
24
24
  "title": "ClampInfo",
25
25
  "type": "object",
26
- "x-contract-version": "0.4.15"
26
+ "x-contract-version": "0.4.18"
27
27
  }
@@ -23,5 +23,5 @@
23
23
  ],
24
24
  "title": "CodeBundle",
25
25
  "type": "object",
26
- "x-contract-version": "0.4.15"
26
+ "x-contract-version": "0.4.18"
27
27
  }
@@ -61,5 +61,5 @@
61
61
  },
62
62
  "title": "Concurrency",
63
63
  "type": "object",
64
- "x-contract-version": "0.4.15"
64
+ "x-contract-version": "0.4.18"
65
65
  }
@@ -55,5 +55,5 @@
55
55
  ],
56
56
  "title": "Conditions",
57
57
  "type": "object",
58
- "x-contract-version": "0.4.15"
58
+ "x-contract-version": "0.4.18"
59
59
  }
@@ -25,5 +25,5 @@
25
25
  ],
26
26
  "title": "ConnectRequest",
27
27
  "type": "object",
28
- "x-contract-version": "0.4.15"
28
+ "x-contract-version": "0.4.18"
29
29
  }
@@ -307,5 +307,5 @@
307
307
  ],
308
308
  "title": "ConnectResponse",
309
309
  "type": "object",
310
- "x-contract-version": "0.4.15"
310
+ "x-contract-version": "0.4.18"
311
311
  }
@@ -18,5 +18,5 @@
18
18
  ],
19
19
  "title": "DeploymentAccepted",
20
20
  "type": "object",
21
- "x-contract-version": "0.4.15"
21
+ "x-contract-version": "0.4.18"
22
22
  }