sequence-base 0.3.17__tar.gz → 0.3.19__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. {sequence_base-0.3.17 → sequence_base-0.3.19}/PKG-INFO +1 -1
  2. {sequence_base-0.3.17 → sequence_base-0.3.19}/pyproject.toml +1 -1
  3. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/constants/version.py +3 -1
  4. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/contract/benchmarks.py +14 -4
  5. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/contract/deployments.py +27 -7
  6. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/contract/eval.py +132 -3
  7. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/contract/flags.py +10 -3
  8. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/contract/policy.py +23 -4
  9. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ActRequest.json +1 -1
  10. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ActResponse.json +1 -1
  11. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ActionChunk.json +1 -1
  12. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ActionContract.json +1 -1
  13. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ActionStep.json +1 -1
  14. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/AdapterTransform.json +1 -1
  15. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/AddLocal.json +1 -1
  16. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/Autoscaler.json +31 -1
  17. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/BenchmarkConditionSpec.json +1 -1
  18. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/BenchmarkRegisterRequest.json +20 -1
  19. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/BenchmarkSpec.json +25 -5
  20. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/BenchmarkStatusResponse.json +1 -1
  21. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/BootstrapResponse.json +1 -1
  22. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/Camera.json +1 -1
  23. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ClampInfo.json +1 -1
  24. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/CodeBundle.json +1 -1
  25. sequence_base-0.3.19/sequence_base/schemas/Concurrency.json +65 -0
  26. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/Conditions.json +1 -1
  27. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ConnectRequest.json +1 -1
  28. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ConnectResponse.json +1 -1
  29. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/DeploymentAccepted.json +1 -1
  30. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/DeploymentPolicy.json +1 -1
  31. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/DeploymentRequest.json +137 -17
  32. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/DeploymentStatusResponse.json +18 -1
  33. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/Embodiment.json +1 -1
  34. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ErrorBody.json +1 -1
  35. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ErrorCodes.json +1 -1
  36. sequence_base-0.3.19/sequence_base/schemas/Estimate.json +52 -0
  37. sequence_base-0.3.19/sequence_base/schemas/EvalDryRunResponse.json +232 -0
  38. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/EvalReport.json +216 -2
  39. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/EvalRequest.json +81 -13
  40. sequence_base-0.3.19/sequence_base/schemas/EvalResponse.json +182 -0
  41. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ImageFrame.json +1 -1
  42. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ImageSpec.json +1 -1
  43. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ImageStepAddLocal.json +1 -1
  44. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ImageStepApt.json +1 -1
  45. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ImageStepEnv.json +1 -1
  46. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ImageStepRun.json +1 -1
  47. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ImageStepUvPip.json +1 -1
  48. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/InferParam.json +1 -1
  49. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/Lease.json +1 -1
  50. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/LossyTransform.json +1 -1
  51. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/MetricAgg.json +1 -1
  52. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/Observation.json +1 -1
  53. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/ObservationContract.json +1 -1
  54. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/PolicyRegistration.json +93 -13
  55. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/PolicySpec.json +84 -10
  56. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/Proprioception.json +1 -1
  57. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/RenderSpec.json +1 -1
  58. sequence_base-0.3.19/sequence_base/schemas/Resources.json +88 -0
  59. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/RolloutResult.json +1 -1
  60. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/SecretCreateRequest.json +1 -1
  61. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/SecretInfo.json +1 -1
  62. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/SecretRef.json +1 -1
  63. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/SeqFlags.json +6 -2
  64. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/Step.json +1 -1
  65. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/SuiteAgg.json +1 -1
  66. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/TaskAgg.json +1 -1
  67. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/Timeouts.json +1 -1
  68. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/UploadTarget.json +1 -1
  69. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/UsageReport.json +1 -1
  70. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/Variant.json +1 -1
  71. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/VolumeAttach.json +1 -1
  72. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/VolumeFile.json +1 -1
  73. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/VolumeInfo.json +1 -1
  74. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/VolumeMount.json +1 -1
  75. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/VolumeRef.json +1 -1
  76. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/VolumeUploadRequest.json +1 -1
  77. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/VolumeUploadResponse.json +1 -1
  78. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schemas/WarmingError.json +1 -1
  79. sequence_base-0.3.17/sequence_base/schemas/Concurrency.json +0 -21
  80. sequence_base-0.3.17/sequence_base/schemas/Estimate.json +0 -25
  81. sequence_base-0.3.17/sequence_base/schemas/EvalDryRunResponse.json +0 -41
  82. sequence_base-0.3.17/sequence_base/schemas/EvalResponse.json +0 -88
  83. sequence_base-0.3.17/sequence_base/schemas/Resources.json +0 -42
  84. {sequence_base-0.3.17 → sequence_base-0.3.19}/.gitignore +0 -0
  85. {sequence_base-0.3.17 → sequence_base-0.3.19}/README.md +0 -0
  86. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/__init__.py +0 -0
  87. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/auth.py +0 -0
  88. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/codec/__init__.py +0 -0
  89. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/codec/image.py +0 -0
  90. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/constants/__init__.py +0 -0
  91. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/constants/env.py +0 -0
  92. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/constants/headers.py +0 -0
  93. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/contract/__init__.py +0 -0
  94. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/contract/accounts.py +0 -0
  95. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/contract/act.py +0 -0
  96. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/contract/connect.py +0 -0
  97. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/contract/errors.py +0 -0
  98. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/contract/io.py +0 -0
  99. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/contract/templates.py +0 -0
  100. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/contract/volumes.py +0 -0
  101. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/contract/wire.py +0 -0
  102. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/errors/__init__.py +0 -0
  103. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/errors/transport.py +0 -0
  104. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/obs/__init__.py +0 -0
  105. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/obs/log.py +0 -0
  106. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/obs/rid.py +0 -0
  107. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/presets/__init__.py +0 -0
  108. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/presets/robot_arm.py +0 -0
  109. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/schema_export.py +0 -0
  110. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/telemetry/__init__.py +0 -0
  111. {sequence_base-0.3.17 → sequence_base-0.3.19}/sequence_base/telemetry/events.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: sequence-base
3
- Version: 0.3.17
3
+ Version: 0.3.19
4
4
  Summary: Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness.
5
5
  Requires-Python: >=3.10
6
6
  Requires-Dist: pydantic>=2.0
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "sequence-base"
3
- version = "0.3.17"
3
+ version = "0.3.19"
4
4
  description = "Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness."
5
5
  # 3.10, not 3.11: `seq` (which imports this) must run on ROS 2 Humble's Python 3.10, and nothing here needs 3.11.
6
6
  # {2026-10-03 the full suite on CPython 3.10.20: "147 PASSED"; vermin --eval-annotations: "MINIMUM REQUIRED VERSIONS: 3.9"}
@@ -4,7 +4,9 @@
4
4
  # across deployments / repos can tell which version of a shape it is validating against. MUST change
5
5
  # whenever any shared wire shape changes (a field added / removed / retyped) — that is what makes
6
6
  # contract evolution mechanically detectable instead of eyeballed. Bump on any shape change.
7
- CONTRACT_VERSION = "0.4.14" # 0.4.13 -> 0.4.14: the /act wire is opaque (ActRequest.inputs / ActResponse.outputs); action / observation contracts optional + DeploymentPolicy/PolicySpec/ConnectResponse .io; the robot-arm types live in sequence_base.presets.robot_arm (contract.act / contract.io re-export them); ImageStepAddLocal (seq.Image.add_local_*); ActionSpace.JOINT_VELOCITY (pi0.5 DROID's joint velocities + gripper position)
7
+ CONTRACT_VERSION = "0.4.16" # 0.4.15 -> 0.4.16: batched benchmarks — SeqFlags.OBSERVE / STEP, BenchmarkSpec / BenchmarkRegisterRequest.envs and .batched; per-arm eval reports — EvalReport.arms / paired / timing / stop_reason, EvalStatus.CANCELED, Estimate.breakdown / default_timing, EvalResponse.arms, EvalDryRunResponse.layout / notes
8
+ # previous: 0.4.15 # 0.4.14 -> 0.4.15: limits count GPUs and CPU/memory are billed like Modal — Resources.cpu / memory_gb optional floats + cpu_limit / memory_limit_gb; Autoscaler.max_gpus / on_full / queue_timeout_s replace max_containers; Concurrency fields optional + replicas_per_gpu / gpus_per_replica; DeploymentStatusResponse.reason; EvalRequest.policy_instances / policy_gpu / benchmark_instances / benchmark_gpu / envs replace gpus / envs_per_gpu; EvalStatus.QUEUED; EvalCards / EvalQueue
9
+ # previous: 0.4.14 # 0.4.13 -> 0.4.14: the /act wire is opaque (ActRequest.inputs / ActResponse.outputs); action / observation contracts optional + DeploymentPolicy/PolicySpec/ConnectResponse .io; the robot-arm types live in sequence_base.presets.robot_arm (contract.act / contract.io re-export them); ImageStepAddLocal (seq.Image.add_local_*); ActionSpace.JOINT_VELOCITY (pi0.5 DROID's joint velocities + gripper position)
8
10
  # previous: 0.4.13 # 0.4.12 -> 0.4.13: template://<name>[@v<n>] — official template weights (parse_template); gs:// / s3:// / https:// still accepted until their removal
9
11
  # previous: 0.4.12 # 0.4.11 -> 0.4.12: one error definition — ErrorBody is every non-2xx body (detail, code, retryable, retry_after_s, request_id, hint, details), ErrorCode covers every case, ERRORS / ErrorCodes.json give each code its status, retryability, SDK class and CLI exit code
10
12
  # previous: 0.4.11 # 0.4.10 -> 0.4.11: weights in named parts (weights_name, extra_weights)
@@ -16,7 +16,7 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator
16
16
 
17
17
  from sequence_base.contract.accounts import SecretRef
18
18
  from sequence_base.contract.deployments import CodeBundle, DeploymentState
19
- from sequence_base.contract.flags import BENCHMARK_BINDABLE, SeqFlags
19
+ from sequence_base.contract.flags import BENCHMARK_BATCHED, BENCHMARK_BINDABLE, SeqFlags
20
20
  from sequence_base.contract.policy import ActionContract, ImageSpec, VolumeRef
21
21
 
22
22
  __all__ = [
@@ -92,8 +92,10 @@ class BenchmarkSpec(BaseModel):
92
92
  conditions: Conditions
93
93
  max_steps: int = Field(..., description="Platform safety cap; truncates and records truncated")
94
94
  handler_ref: str
95
- bindings: dict[SeqFlags, str] = Field(..., description="SETUP / RESET / CHECK / SCORE → method name")
95
+ bindings: dict[SeqFlags, str] = Field(..., description="SETUP / RESET / CHECK / SCORE (+ OBSERVE / STEP) → method name")
96
96
  gpu: str | list[str] | None = None
97
+ # How many environments one benchmark instance runs at once when an eval does not say (the author's default).
98
+ envs: int | None = Field(default=None, ge=1, le=65536)
97
99
  assets: list[VolumeRef] = Field(default_factory=list)
98
100
  render: RenderSpec | None = None
99
101
  cpu: int | None = None
@@ -108,9 +110,12 @@ class BenchmarkSpec(BaseModel):
108
110
  if bad:
109
111
  names = ", ".join(sorted(f.name or str(f.value) for f in bad))
110
112
  raise ValueError(
111
- f"benchmark bindings may only cover SETUP / RESET / CHECK / SCORE; got disallowed {names}"
113
+ f"benchmark bindings may only cover SETUP / RESET / CHECK / SCORE / OBSERVE / STEP; got disallowed {names}"
112
114
  " (ROLLOUT is deprecated and must not be bound)"
113
115
  )
116
+ batched = set(self.bindings) & set(BENCHMARK_BATCHED)
117
+ if batched and batched != set(BENCHMARK_BATCHED):
118
+ raise ValueError("a batched benchmark binds both OBSERVE and STEP")
114
119
  return self
115
120
 
116
121
 
@@ -166,8 +171,13 @@ class BenchmarkRegisterRequest(BaseModel):
166
171
  timeout_s: float | None = Field(default=None, gt=0)
167
172
  # The adapter (module:Class in the benchmark's own code) an eval uses when it names none.
168
173
  default_adapter_ref: str | None = None
169
- # A GPU for each sim sandbox (a renderer or a GPU physics engine): a card name the platform offers. None: CPU.
174
+ # The card each benchmark instance runs on (L4 or RTX-PRO-6000); None takes the platform's default (L4). The
175
+ # simulation never runs on a CPU. {gs-plans/benchmark-internals/design.md "一、规则"}
170
176
  gpu: str | None = Field(default=None, max_length=32)
177
+ # How many environments one instance runs at once when an eval does not say; None takes 1.
178
+ envs: int | None = Field(default=None, ge=1, le=65536)
179
+ # Whether the benchmark is batched (binds OBSERVE and STEP and steps many environments in one call).
180
+ batched: bool = False
171
181
  # The Dockerfile's build context, as for a deployment.
172
182
  image_context: CodeBundle | None = None
173
183
 
@@ -52,12 +52,24 @@ class Resources(BaseModel):
52
52
  model_config = ConfigDict(extra="forbid")
53
53
 
54
54
  gpu: list[str]
55
- cpu: int
56
- memory_gb: int
55
+ # Physical cores and GiB the worker requests, billed like Modal: per second, at the greater of the request and what
56
+ # it uses. None requests the minimum (0.125 cores, 0.125 GiB) and the worker may use more. A limit caps what it may
57
+ # use; None leaves it free to burst. {gs-plans/parallel/cpu-memory.md "4. 默认申请量照 Modal"}
58
+ cpu: float | None = Field(default=None, ge=0.125, le=64)
59
+ cpu_limit: float | None = Field(default=None, ge=0.125, le=128)
60
+ memory_gb: float | None = Field(default=None, ge=0.125, le=1024)
61
+ memory_limit_gb: float | None = Field(default=None, ge=0.125, le=1024)
57
62
  # Where the workers run (near the robots): one of the regions the platform offers. None: the platform picks.
58
63
  # {gs-plans/customization/policy.md "区域"}
59
64
  region: str | None = Field(default=None, max_length=32)
60
65
 
66
+ @model_validator(mode="after")
67
+ def _limits_cover_requests(self) -> "Resources":
68
+ for what, req, lim in (("cpu", self.cpu, self.cpu_limit), ("memory", self.memory_gb, self.memory_limit_gb)):
69
+ if req is not None and lim is not None and lim < req:
70
+ raise ValueError(f"the {what} limit must be at least the {what} request")
71
+ return self
72
+
61
73
 
62
74
  class Autoscaler(BaseModel):
63
75
  """How the deployment scales. Defaults mirror PolicySpec so an omitted block scales to zero."""
@@ -65,9 +77,18 @@ class Autoscaler(BaseModel):
65
77
  model_config = ConfigDict(extra="forbid")
66
78
 
67
79
  min_containers: int = 0
80
+ # Accepted and ignored: limits count GPUs (`max_gpus`). sequence-ai <= 0.28 still sends it and does not pin
81
+ # sequence-base, so removing it would break their deploys; it goes when no supported SDK sends it.
68
82
  max_containers: int | None = None
83
+ # The most GPUs this deployment may hold at once. None: no cap of its own — only the account's and the platform's
84
+ # GPU limits bound it. Limits count cards, never machines. {gs-plans/parallel/serving.md "1. 限制按卡数,不按台数"}
85
+ max_gpus: int | None = Field(default=None, ge=1, le=256)
69
86
  buffer_containers: int | None = None
70
87
  scaledown_window: int = 600
88
+ # When every slot is taken and no card is left: queue the new robot before its control loop starts, or refuse it at
89
+ # once; how long a queued robot waits before it is told why. {gs-plans/parallel/serving-queue.md "connect 的处理顺序"}
90
+ on_full: Literal["queue", "reject"] = "queue"
91
+ queue_timeout_s: int = Field(default=120, ge=5, le=3600)
71
92
 
72
93
 
73
94
  class Timeouts(BaseModel):
@@ -205,11 +226,8 @@ class DeploymentRequest(BaseModel):
205
226
  # both key and value; the gateway only stores the dump, no consumer needs dict lookup.
206
227
  volumes: list[VolumeAttach] = Field(default_factory=list)
207
228
  autoscaler: Autoscaler = Field(default_factory=Autoscaler)
208
- # An omitted @seq.concurrent must NOT send null: the gateway would then have no concurrency to scale by.
209
- # Materialize the spec default — contracts.md §2's example {"max_inputs": 1, "target_inputs": 1}, i.e.
210
- # one input per container — same discipline as `autoscaler`
211
- # (goal 13).
212
- concurrency: Concurrency = Field(default_factory=lambda: Concurrency(max_inputs=1, target_inputs=1))
229
+ # Every field optional: an omitted @seq.concurrent serves one robot session per model copy, one copy per card.
230
+ concurrency: Concurrency = Field(default_factory=Concurrency)
213
231
  timeouts: Timeouts | None = None
214
232
  # Weights in several named parts: `weights` is the first and `weights_name` its name; `extra_weights` the others,
215
233
  # each provisioned like `weights`. @seq.load then receives {name: directory}. Both None (and absent from the wire):
@@ -283,3 +301,5 @@ class DeploymentStatusResponse(BaseModel):
283
301
  status: DeploymentState
284
302
  model: str
285
303
  error: str | None = Field(default=None, description="Build-failure reason; set only when FAILED")
304
+ # Why a deployment that was serving was marked FAILED: its workers kept dying of the same cause.
305
+ reason: Literal["gpu_out_of_memory", "host_out_of_memory", "crashed"] | None = None
@@ -17,9 +17,16 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator
17
17
  __all__ = [
18
18
  "EvalRequest",
19
19
  "ClampInfo",
20
+ "ArmReport",
21
+ "PairedDiff",
22
+ "EvalTiming",
20
23
  "Estimate",
21
24
  "EvalResponse",
22
25
  "EvalDryRunResponse",
26
+ "CardSide",
27
+ "CardInUse",
28
+ "EvalCards",
29
+ "EvalQueue",
23
30
  "RolloutResult",
24
31
  "EvalStatus",
25
32
  "SuiteAgg",
@@ -42,11 +49,21 @@ class EvalRequest(BaseModel):
42
49
  benchmark: str
43
50
  suite: str | None = None
44
51
  trials: int | None = None
45
- gpus: int | None = None
46
- envs_per_gpu: int | None = None
52
+ # How the eval is laid out, written by the user and never derived: how many policy machines and on which card
53
+ # (default: one, on the deployment's declared card), how many benchmark machines and on which card (default: one,
54
+ # on the benchmark's declared card), and how many environments each benchmark machine runs (default: the
55
+ # benchmark author's `envs=`, else 1). {gs-plans/parallel/eval.md "1. 两边写法对齐"}
56
+ policy_instances: int = Field(default=1, ge=1, le=64)
57
+ policy_gpu: str | None = Field(default=None, max_length=32)
58
+ benchmark_instances: int = Field(default=1, ge=1, le=64)
59
+ benchmark_gpu: str | None = Field(default=None, max_length=32)
60
+ envs: int | None = Field(default=None, ge=1, le=100000)
47
61
  vlm_max_concurrency: int | None = None
48
62
  adapter: str | None = None
49
- compare: str | None = Field(default=None, description="Another model id to compare against")
63
+ # Arms run side by side in one eval: other models on the same episodes (each its own policy instances), and/or a
64
+ # grid of inference-parameter values (every combination is one arm).
65
+ compare: str | list[str] | None = Field(default=None, description="Other model ids to compare against")
66
+ sweep: dict[str, list[int | float | bool | str]] | None = Field(default=None, max_length=8)
50
67
  dry_run: bool = False
51
68
  # Only these tasks (of the selected suite / trials). {gs-plans/customization/benchmark.md "子集"}
52
69
  tasks: list[str] | None = Field(default=None, max_length=1000)
@@ -79,6 +96,10 @@ class Estimate(BaseModel):
79
96
  wall_clock_s: float
80
97
  cost_usd: float
81
98
  vlm_calls: int
99
+ # The cost split by machine and by resource (GPU / CPU / memory), when the gateway itemizes it.
100
+ breakdown: dict[str, float] | None = None
101
+ # True when no run of this benchmark version has been measured yet and the estimate uses the default step time.
102
+ default_timing: bool | None = None
82
103
 
83
104
 
84
105
  class EvalResponse(BaseModel):
@@ -90,6 +111,58 @@ class EvalResponse(BaseModel):
90
111
  gpus_granted: int
91
112
  estimate: Estimate
92
113
  clamped_by: ClampInfo | None = None
114
+ # Set when the account was short of GPUs: the eval waits (up to its timeout) instead of being refused.
115
+ queue: EvalQueue | None = None
116
+ # How many arms (model x swept parameters) the eval runs side by side.
117
+ arms: int | None = None
118
+
119
+
120
+ class CardSide(BaseModel):
121
+ """One side of an eval's machines: how many, on which card, and how many GPUs each holds."""
122
+
123
+ model_config = ConfigDict(extra="forbid")
124
+
125
+ instances: int
126
+ gpu: str
127
+ cards_each: int
128
+
129
+
130
+ class CardInUse(BaseModel):
131
+ """The account's GPUs held now, serving and evals apart; None when it could not be read."""
132
+
133
+ model_config = ConfigDict(extra="forbid")
134
+
135
+ serving: int | None = None
136
+ eval: int | None = None
137
+
138
+
139
+ class EvalCards(BaseModel):
140
+ """The dry-run's card report — the same checks a real submit runs, without starting or charging anything.
141
+ {gs-plans/parallel/eval.md "4. 预演里卡的部分"}"""
142
+
143
+ model_config = ConfigDict(extra="forbid")
144
+
145
+ policy: CardSide
146
+ benchmark: CardSide
147
+ total: int
148
+ limit: int
149
+ in_use: CardInUse
150
+ available: int | None = None
151
+ short: int = 0
152
+ reason: Literal["gpu_limit_reached", "gpu_busy", "gpu_low_stock"] | None = None
153
+ would_queue: bool = False
154
+ queue_timeout_s: int = 600
155
+
156
+
157
+ class EvalQueue(BaseModel):
158
+ """An eval waiting for the account's GPUs: what it needs, what is free, how long it has waited, when it gives up."""
159
+
160
+ model_config = ConfigDict(extra="forbid")
161
+
162
+ cards: int
163
+ available: int | None = None
164
+ waited_s: float = 0
165
+ gives_up_at: str | None = None
93
166
 
94
167
 
95
168
  class EvalDryRunResponse(BaseModel):
@@ -98,6 +171,11 @@ class EvalDryRunResponse(BaseModel):
98
171
  model_config = ConfigDict(extra="forbid")
99
172
 
100
173
  estimate: Estimate
174
+ cards: EvalCards | None = None
175
+ # The eval's layout as it would run: per side the instances, card and environments (the user's numbers or the
176
+ # fixed defaults), and the arms. Advice only ever comes as `notes`; the platform never changes the numbers.
177
+ layout: dict | None = None
178
+ notes: list[str] = Field(default_factory=list)
101
179
 
102
180
 
103
181
  class RolloutResult(BaseModel):
@@ -124,9 +202,11 @@ class EvalStatus(str, Enum):
124
202
  """Restricted enum — same discipline as DeploymentState (not a free string)."""
125
203
 
126
204
  PENDING = "pending"
205
+ QUEUED = "queued"
127
206
  RUNNING = "running"
128
207
  COMPLETED = "completed"
129
208
  FAILED = "failed"
209
+ CANCELED = "canceled"
130
210
 
131
211
 
132
212
  class SuiteAgg(BaseModel):
@@ -173,6 +253,49 @@ class LossyTransform(BaseModel):
173
253
  reason: str
174
254
 
175
255
 
256
+ class ArmReport(BaseModel):
257
+ """One arm of an eval (a model x one combination of swept inference parameters): its own metrics and episode
258
+ counts, so arms are compared side by side and never pooled."""
259
+
260
+ model_config = ConfigDict(extra="forbid")
261
+
262
+ arm: int
263
+ model: str
264
+ params: dict[str, int | float | bool | str] = Field(default_factory=dict)
265
+ metrics: dict[str, MetricAgg] = Field(default_factory=dict)
266
+ # Each metric's 95% confidence interval as [low, high].
267
+ intervals: dict[str, list[float]] = Field(default_factory=dict)
268
+ episodes: int = 0
269
+ failed: int = 0
270
+ not_run: int = 0
271
+ budget_stopped: bool = False
272
+
273
+
274
+ class PairedDiff(BaseModel):
275
+ """`compare`: one metric's difference between two arms over the seeds both ran (b - a), with its interval."""
276
+
277
+ model_config = ConfigDict(extra="forbid")
278
+
279
+ metric: str
280
+ arm_a: int
281
+ arm_b: int
282
+ n: int
283
+ mean_diff: float
284
+ interval: list[float]
285
+
286
+
287
+ class EvalTiming(BaseModel):
288
+ """Where an eval's time went: cold starts, and per step the simulation, rendering, the policy's batched forward and
289
+ the network around it, plus the forward's average batch size."""
290
+
291
+ model_config = ConfigDict(extra="forbid")
292
+
293
+ cold_start_s: dict[str, float] = Field(default_factory=dict)
294
+ step_ms: dict[str, float] = Field(default_factory=dict)
295
+ batch_size_mean: float | None = None
296
+ steps: int = 0
297
+
298
+
176
299
  class EvalReport(BaseModel):
177
300
  """GET /v1/eval/{id} (eval-raw revision). NO hard-coded `overall_sr` / `success_rate` top-level field —
178
301
  `metrics` is a passthrough dict keyed by the user's metric names (from `@seq.check`'s `Step.metrics`),
@@ -194,6 +317,12 @@ class EvalReport(BaseModel):
194
317
  errors: list[str] = Field(
195
318
  default_factory=list, description="Why episodes or the run failed, so an incomplete report says why"
196
319
  )
320
+ # Per arm (one entry for an eval with one arm), paired comparisons for `compare`, the timing breakdown, and why
321
+ # the run stopped early (canceled / budget), when it did.
322
+ arms: list[ArmReport] | None = None
323
+ paired: list[PairedDiff] | None = None
324
+ timing: EvalTiming | None = None
325
+ stop_reason: str | None = None
197
326
 
198
327
 
199
328
  class AdapterKind(str, Enum):
@@ -41,18 +41,25 @@ class SeqFlags(IntFlag):
41
41
  SCORE = 16384 # cross-episode aggregate (optional; default = mean of each numeric metric)
42
42
  RESET = 32768 # build one episode's env (gym-style handle) — eval-raw new hook, bit 15
43
43
  CHECK = 65536 # per step: read raw env state → done + user-named metrics — eval-raw new hook, bit 16
44
+ # A batched benchmark (a GPU sim that steps many environments at once) also binds these two; its RESET then
45
+ # takes (slots, conditions) and its CHECK (slots). {gs-plans/benchmark-internals/design.md "三、两种写法"}
46
+ OBSERVE = 131072 # read the observation of the given slots, one per slot — bit 17
47
+ STEP = 262144 # apply one action to each given slot — bit 18
44
48
  # DEPRECATED — the old single-hook benchmark model. Bit 8192 stays reserved (never reused for another
45
49
  # member) but is NOT bound in any registration; setup/reset/check/score replaces it. Kept only so the
46
50
  # value cannot be recycled and an old serialized 8192 still decodes to a known name.
47
51
  ROLLOUT = 8192
48
52
 
49
53
 
50
- # The benchmark lifecycle bits a live registration may bind (eval-raw): SETUP / RESET / CHECK / SCORE.
51
- # ROLLOUT is deliberately absent — deprecated, never bound.
54
+ # The benchmark lifecycle bits a live registration may bind (eval-raw): SETUP / RESET / CHECK / SCORE, and OBSERVE /
55
+ # STEP for a batched benchmark. ROLLOUT is deliberately absent — deprecated, never bound.
52
56
  BENCHMARK_BINDABLE: frozenset[SeqFlags] = frozenset(
53
- {SeqFlags.SETUP, SeqFlags.RESET, SeqFlags.CHECK, SeqFlags.SCORE}
57
+ {SeqFlags.SETUP, SeqFlags.RESET, SeqFlags.CHECK, SeqFlags.SCORE, SeqFlags.OBSERVE, SeqFlags.STEP}
54
58
  )
55
59
 
60
+ # The two bits that make a benchmark batched: present together or not at all.
61
+ BENCHMARK_BATCHED: frozenset[SeqFlags] = frozenset({SeqFlags.OBSERVE, SeqFlags.STEP})
62
+
56
63
  # Deprecated benchmark bits: reserved value, must not appear in any bindings.
57
64
  DEPRECATED_FLAGS: frozenset[SeqFlags] = frozenset({SeqFlags.ROLLOUT})
58
65
 
@@ -10,7 +10,7 @@ from __future__ import annotations
10
10
 
11
11
  import re
12
12
 
13
- from typing import Any
13
+ from typing import Any, Literal
14
14
 
15
15
  from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
16
16
 
@@ -153,12 +153,26 @@ class VolumeRef(BaseModel):
153
153
 
154
154
 
155
155
  class Concurrency(BaseModel):
156
- """How many inputs one container serves at once: the most it takes, and the level it scales to hold. Both required."""
156
+ """How one worker serves robots, every field optional. `replicas_per_gpu` model copies share each card (each copy is
157
+ its own process and serves one control loop); `gpus_per_replica` cards make one copy (None: every card of the
158
+ worker). A worker's slots = cards // gpus_per_replica × replicas_per_gpu. `target_inputs` is how many robot sessions
159
+ a worker holds before another starts, `max_inputs` the most it takes while a new one boots — both default to the
160
+ slots. {gs-plans/parallel/serving.md "2. 一台 worker 里" / "3. 一台 worker 多张卡"}"""
157
161
 
158
162
  model_config = ConfigDict(extra="forbid")
159
163
 
160
- max_inputs: int
161
- target_inputs: int
164
+ max_inputs: int | None = Field(default=None, ge=1, le=1024)
165
+ target_inputs: int | None = Field(default=None, ge=1, le=1024)
166
+ replicas_per_gpu: int | None = Field(default=None, ge=1, le=64)
167
+ gpus_per_replica: int | None = Field(default=None, ge=1, le=256)
168
+
169
+ @model_validator(mode="after")
170
+ def _consistent(self) -> "Concurrency":
171
+ if self.max_inputs is not None and self.target_inputs is not None and self.max_inputs < self.target_inputs:
172
+ raise ValueError("max_inputs must be at least target_inputs")
173
+ if (self.gpus_per_replica or 1) > 1 and (self.replicas_per_gpu or 1) > 1:
174
+ raise ValueError("a copy that spans several cards is one copy per card group: replicas_per_gpu must be 1")
175
+ return self
162
176
 
163
177
 
164
178
  class PolicySpec(BaseModel):
@@ -176,9 +190,14 @@ class PolicySpec(BaseModel):
176
190
  volumes: dict[str, VolumeRef] = Field(default_factory=dict)
177
191
  secrets: list[SecretRef] = Field(default_factory=list)
178
192
  min_containers: int = 0
193
+ # Accepted and ignored: limits count GPUs (`max_gpus`). sequence-ai <= 0.28 still sends it and does not pin
194
+ # sequence-base, so removing it would break their deploys; it goes when no supported SDK sends it.
179
195
  max_containers: int | None = None
196
+ max_gpus: int | None = Field(default=None, ge=1, le=256)
180
197
  buffer_containers: int | None = None
181
198
  scaledown_window: int = 600
199
+ on_full: Literal["queue", "reject"] = "queue"
200
+ queue_timeout_s: int = Field(default=120, ge=5, le=3600)
182
201
  concurrency: Concurrency | None = None
183
202
  streaming: bool = False
184
203
  frames_per_chunk: int | None = None
@@ -65,5 +65,5 @@
65
65
  ],
66
66
  "title": "ActRequest",
67
67
  "type": "object",
68
- "x-contract-version": "0.4.14"
68
+ "x-contract-version": "0.4.16"
69
69
  }
@@ -75,5 +75,5 @@
75
75
  ],
76
76
  "title": "ActResponse",
77
77
  "type": "object",
78
- "x-contract-version": "0.4.14"
78
+ "x-contract-version": "0.4.16"
79
79
  }
@@ -99,5 +99,5 @@
99
99
  ],
100
100
  "title": "ActionChunk",
101
101
  "type": "object",
102
- "x-contract-version": "0.4.14"
102
+ "x-contract-version": "0.4.16"
103
103
  }
@@ -100,5 +100,5 @@
100
100
  ],
101
101
  "title": "ActionContract",
102
102
  "type": "object",
103
- "x-contract-version": "0.4.14"
103
+ "x-contract-version": "0.4.16"
104
104
  }
@@ -19,5 +19,5 @@
19
19
  ],
20
20
  "title": "ActionStep",
21
21
  "type": "object",
22
- "x-contract-version": "0.4.14"
22
+ "x-contract-version": "0.4.16"
23
23
  }
@@ -38,5 +38,5 @@
38
38
  ],
39
39
  "title": "AdapterTransform",
40
40
  "type": "object",
41
- "x-contract-version": "0.4.14"
41
+ "x-contract-version": "0.4.16"
42
42
  }
@@ -23,5 +23,5 @@
23
23
  ],
24
24
  "title": "AddLocal",
25
25
  "type": "object",
26
- "x-contract-version": "0.4.14"
26
+ "x-contract-version": "0.4.16"
27
27
  }
@@ -26,11 +26,41 @@
26
26
  "default": null,
27
27
  "title": "Max Containers"
28
28
  },
29
+ "max_gpus": {
30
+ "anyOf": [
31
+ {
32
+ "maximum": 256,
33
+ "minimum": 1,
34
+ "type": "integer"
35
+ },
36
+ {
37
+ "type": "null"
38
+ }
39
+ ],
40
+ "default": null,
41
+ "title": "Max Gpus"
42
+ },
29
43
  "min_containers": {
30
44
  "default": 0,
31
45
  "title": "Min Containers",
32
46
  "type": "integer"
33
47
  },
48
+ "on_full": {
49
+ "default": "queue",
50
+ "enum": [
51
+ "queue",
52
+ "reject"
53
+ ],
54
+ "title": "On Full",
55
+ "type": "string"
56
+ },
57
+ "queue_timeout_s": {
58
+ "default": 120,
59
+ "maximum": 3600,
60
+ "minimum": 5,
61
+ "title": "Queue Timeout S",
62
+ "type": "integer"
63
+ },
34
64
  "scaledown_window": {
35
65
  "default": 600,
36
66
  "title": "Scaledown Window",
@@ -39,5 +69,5 @@
39
69
  },
40
70
  "title": "Autoscaler",
41
71
  "type": "object",
42
- "x-contract-version": "0.4.14"
72
+ "x-contract-version": "0.4.16"
43
73
  }
@@ -62,5 +62,5 @@
62
62
  ],
63
63
  "title": "BenchmarkConditionSpec",
64
64
  "type": "object",
65
- "x-contract-version": "0.4.14"
65
+ "x-contract-version": "0.4.16"
66
66
  }
@@ -523,6 +523,11 @@
523
523
  "title": "Assets",
524
524
  "type": "array"
525
525
  },
526
+ "batched": {
527
+ "default": false,
528
+ "title": "Batched",
529
+ "type": "boolean"
530
+ },
526
531
  "code": {
527
532
  "anyOf": [
528
533
  {
@@ -577,6 +582,20 @@
577
582
  "default": null,
578
583
  "title": "Default Adapter Ref"
579
584
  },
585
+ "envs": {
586
+ "anyOf": [
587
+ {
588
+ "maximum": 65536,
589
+ "minimum": 1,
590
+ "type": "integer"
591
+ },
592
+ {
593
+ "type": "null"
594
+ }
595
+ ],
596
+ "default": null,
597
+ "title": "Envs"
598
+ },
580
599
  "gpu": {
581
600
  "anyOf": [
582
601
  {
@@ -681,5 +700,5 @@
681
700
  ],
682
701
  "title": "BenchmarkRegisterRequest",
683
702
  "type": "object",
684
- "x-contract-version": "0.4.14"
703
+ "x-contract-version": "0.4.16"
685
704
  }
@@ -273,15 +273,18 @@
273
273
  8192,
274
274
  16384,
275
275
  32768,
276
- 65536
276
+ 65536,
277
+ 131072,
278
+ 262144
277
279
  ],
278
280
  "title": "SeqFlags",
279
281
  "type": "integer",
280
- "x-contract-version": "0.4.14",
282
+ "x-contract-version": "0.4.16",
281
283
  "x-flag-names": {
282
284
  "CHECK": 65536,
283
285
  "INFER": 16,
284
286
  "LOAD": 2,
287
+ "OBSERVE": 131072,
285
288
  "ON_EPISODE": 1024,
286
289
  "PLAN": 4,
287
290
  "RESET": 32768,
@@ -289,6 +292,7 @@
289
292
  "SCORE": 16384,
290
293
  "SETUP": 4096,
291
294
  "SHUTDOWN": 8,
295
+ "STEP": 262144,
292
296
  "WARMUP": 1
293
297
  }
294
298
  },
@@ -357,7 +361,7 @@
357
361
  "additionalProperties": {
358
362
  "type": "string"
359
363
  },
360
- "description": "SETUP / RESET / CHECK / SCORE → method name",
364
+ "description": "SETUP / RESET / CHECK / SCORE (+ OBSERVE / STEP) → method name",
361
365
  "propertyNames": {
362
366
  "enum": [
363
367
  "1",
@@ -370,7 +374,9 @@
370
374
  "8192",
371
375
  "16384",
372
376
  "32768",
373
- "65536"
377
+ "65536",
378
+ "131072",
379
+ "262144"
374
380
  ],
375
381
  "type": "string"
376
382
  },
@@ -392,6 +398,20 @@
392
398
  "default": null,
393
399
  "title": "Cpu"
394
400
  },
401
+ "envs": {
402
+ "anyOf": [
403
+ {
404
+ "maximum": 65536,
405
+ "minimum": 1,
406
+ "type": "integer"
407
+ },
408
+ {
409
+ "type": "null"
410
+ }
411
+ ],
412
+ "default": null,
413
+ "title": "Envs"
414
+ },
395
415
  "gpu": {
396
416
  "anyOf": [
397
417
  {
@@ -467,5 +487,5 @@
467
487
  ],
468
488
  "title": "BenchmarkSpec",
469
489
  "type": "object",
470
- "x-contract-version": "0.4.14"
490
+ "x-contract-version": "0.4.16"
471
491
  }