sequence-base 0.3.22__tar.gz → 0.3.24__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sequence_base-0.3.22 → sequence_base-0.3.24}/PKG-INFO +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/pyproject.toml +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/constants/version.py +3 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/contract/benchmarks.py +12 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/contract/deployments.py +41 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/contract/eval.py +35 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ActRequest.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ActResponse.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ActionChunk.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ActionContract.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ActionStep.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/AdapterTransform.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/AddLocal.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/Autoscaler.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/BenchmarkConditionSpec.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/BenchmarkRegisterRequest.json +119 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/BenchmarkSpec.json +2 -2
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/BenchmarkStatusResponse.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/BootstrapResponse.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/Camera.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ClampInfo.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/CodeBundle.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/Concurrency.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/Conditions.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ConnectRequest.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ConnectResponse.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/DeploymentAccepted.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/DeploymentPolicy.json +62 -2
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/DeploymentRequest.json +62 -2
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/DeploymentStatusResponse.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/Embodiment.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ErrorBody.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ErrorCodes.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/Estimate.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/EvalDryRunResponse.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/EvalReport.json +126 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/EvalRequest.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/EvalResponse.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ImageFrame.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ImageSpec.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ImageStepAddLocal.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ImageStepApt.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ImageStepEnv.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ImageStepRun.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ImageStepUvPip.json +1 -1
- sequence_base-0.3.24/sequence_base/schemas/InferParam.json +103 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/Lease.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/LossyTransform.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/MetricAgg.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/Observation.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/ObservationContract.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/PolicyRegistration.json +2 -2
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/PolicySpec.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/Proprioception.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/RenderSpec.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/Resources.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/RolloutResult.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/SecretCreateRequest.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/SecretInfo.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/SecretRef.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/SeqFlags.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/Step.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/SuiteAgg.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/TaskAgg.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/Timeouts.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/UploadTarget.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/UsageReport.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/Variant.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/VolumeAttach.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/VolumeFile.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/VolumeInfo.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/VolumeMount.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/VolumeRef.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/VolumeUploadRequest.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/VolumeUploadResponse.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/WarmingError.json +1 -1
- sequence_base-0.3.22/sequence_base/schemas/InferParam.json +0 -43
- {sequence_base-0.3.22 → sequence_base-0.3.24}/.gitignore +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/README.md +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/__init__.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/auth.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/codec/__init__.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/codec/image.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/constants/__init__.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/constants/env.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/constants/headers.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/contract/__init__.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/contract/accounts.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/contract/act.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/contract/connect.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/contract/errors.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/contract/flags.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/contract/io.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/contract/policy.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/contract/templates.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/contract/volumes.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/contract/wire.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/errors/__init__.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/errors/transport.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/obs/__init__.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/obs/log.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/obs/rid.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/presets/__init__.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/presets/robot_arm.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schema_export.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/telemetry/__init__.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/telemetry/events.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sequence-base
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.24
|
|
4
4
|
Summary: Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness.
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Requires-Dist: pydantic>=2.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sequence-base"
|
|
3
|
-
version = "0.3.
|
|
3
|
+
version = "0.3.24"
|
|
4
4
|
description = "Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness."
|
|
5
5
|
# 3.10, not 3.11: `seq` (which imports this) must run on ROS 2 Humble's Python 3.10, and nothing here needs 3.11.
|
|
6
6
|
# {2026-10-03 the full suite on CPython 3.10.20: "147 PASSED"; vermin --eval-annotations: "MINIMUM REQUIRED VERSIONS: 3.9"}
|
|
@@ -4,7 +4,9 @@
|
|
|
4
4
|
# across deployments / repos can tell which version of a shape it is validating against. MUST change
|
|
5
5
|
# whenever any shared wire shape changes (a field added / removed / retyped) — that is what makes
|
|
6
6
|
# contract evolution mechanically detectable instead of eyeballed. Bump on any shape change.
|
|
7
|
-
CONTRACT_VERSION = "0.4.
|
|
7
|
+
CONTRACT_VERSION = "0.4.21" # 0.4.20 -> 0.4.21: InferParam.min / max / choices — the values an inference parameter allows (seq.Range / seq.OneOf), refused before inference outside them; None, left off the wire, as before (gs-plans deploy-serving/35)
|
|
8
|
+
# previous: 0.4.20 # 0.4.19 -> 0.4.20: BenchmarkRegisterRequest.params — the parameters a benchmark's @seq.setup declares (InferParam: type and default), which an eval may set; EvalReport.arm_params — per arm each parameter's value and source, and the steps that asked the policy of all it took (gs-plans eval-adapters/36)
|
|
9
|
+
# previous: 0.4.19 # 0.4.18 -> 0.4.19: RenderSpec.flip — how the platform turns a recorded camera's frames before encoding ("vertical" / "horizontal" / "both"; None, left off the wire, as drawn), declared once by the benchmark's author (seq.Record(flip=...)); gs-plans benchmarks
|
|
8
10
|
# previous: 0.4.18 # 0.4.17 -> 0.4.18: DeploymentPolicy.models — the platform models a policy calls while it runs (catalogue ids), which the act key driving it may call on /v1/chat and /v1/decide (gs-plans models-api/11)
|
|
9
11
|
# previous: 0.4.17 # 0.4.16 -> 0.4.17: weights optional — PolicySpec.weights, DeploymentRequest.weights and BootstrapResponse.weights_uri may be None, for a policy without a checkpoint (nothing copied or mounted; gs-plans storage T13)
|
|
10
12
|
# previous: 0.4.16 # 0.4.15 -> 0.4.16: batched benchmarks — SeqFlags.OBSERVE / STEP, BenchmarkSpec / BenchmarkRegisterRequest.envs and .batched; per-arm eval reports — EvalReport.arms / paired / timing / stop_reason, EvalStatus.CANCELED, Estimate.breakdown / default_timing, EvalResponse.arms, EvalDryRunResponse.layout / notes
|
|
@@ -17,7 +17,7 @@ from typing import Literal
|
|
|
17
17
|
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
18
18
|
|
|
19
19
|
from sequence_base.contract.accounts import SecretRef
|
|
20
|
-
from sequence_base.contract.deployments import CodeBundle, DeploymentState
|
|
20
|
+
from sequence_base.contract.deployments import CodeBundle, DeploymentState, InferParam
|
|
21
21
|
from sequence_base.contract.flags import BENCHMARK_BATCHED, BENCHMARK_BINDABLE, SeqFlags
|
|
22
22
|
from sequence_base.contract.policy import ActionContract, ImageSpec, VolumeRef
|
|
23
23
|
|
|
@@ -185,11 +185,22 @@ class BenchmarkRegisterRequest(BaseModel):
|
|
|
185
185
|
gpu: str | None = Field(default=None, max_length=32)
|
|
186
186
|
# How many environments one instance runs at once when an eval does not say; None takes 1.
|
|
187
187
|
envs: int | None = Field(default=None, ge=1, le=65536)
|
|
188
|
+
# The parameters its @seq.setup declares — `def setup(self, envs, *, noise_std: float = 0.02)` — each with its type
|
|
189
|
+
# and default, which an eval may set (--param benchmark.noise_std=0); None: it declares none, or an SDK from before
|
|
190
|
+
# them. {gs-plans/eval-adapters/36}
|
|
191
|
+
params: dict[str, InferParam] | None = Field(default=None, max_length=32)
|
|
188
192
|
# Whether the benchmark is batched (binds OBSERVE and STEP and steps many environments in one call).
|
|
189
193
|
batched: bool = False
|
|
190
194
|
# The Dockerfile's build context, as for a deployment.
|
|
191
195
|
image_context: CodeBundle | None = None
|
|
192
196
|
|
|
197
|
+
@model_validator(mode="after")
|
|
198
|
+
def _param_names(self) -> "BenchmarkRegisterRequest":
|
|
199
|
+
for name in self.params or {}:
|
|
200
|
+
if not name.isidentifier() or name.startswith("_"):
|
|
201
|
+
raise ValueError(f"setup parameter {name!r} is not a public Python identifier")
|
|
202
|
+
return self
|
|
203
|
+
|
|
193
204
|
|
|
194
205
|
class BenchmarkStatusResponse(BaseModel):
|
|
195
206
|
"""GET /v1/benchmarks/{name} — the build state of a registered benchmark (same lifecycle as a deployment:
|
|
@@ -116,12 +116,52 @@ class VolumeAttach(BaseModel):
|
|
|
116
116
|
class InferParam(BaseModel):
|
|
117
117
|
"""One inference parameter the policy's `@seq.infer` declares — `def infer(self, obs, *, num_steps: int = 10)`
|
|
118
118
|
— with its type and default. An /act or an eval may set it; the worker passes only declared names, cast to
|
|
119
|
-
this type, and refuses any other. {gs-plans/customization/policy.md "四、推理行为"}
|
|
119
|
+
this type, and refuses any other. {gs-plans/customization/policy.md "四、推理行为"}
|
|
120
|
+
|
|
121
|
+
`min` / `max` (a number's bounds, inclusive) and `choices` (the values it may take) — declared as
|
|
122
|
+
`Annotated[int, seq.Range(1, 10)]` / `seq.OneOf(...)` — refuse any other value before inference, saying which are
|
|
123
|
+
allowed; None (left off the wire) holds it to its type alone. {gs-plans/deploy-serving/35 "允许声明范围或可选值"}"""
|
|
120
124
|
|
|
121
125
|
model_config = ConfigDict(extra="forbid")
|
|
122
126
|
|
|
123
127
|
type: Literal["int", "float", "bool", "str"]
|
|
124
128
|
default: int | float | bool | str | None = None
|
|
129
|
+
min: int | float | None = None
|
|
130
|
+
max: int | float | None = None
|
|
131
|
+
choices: list[int | float | bool | str] | None = Field(default=None, min_length=1, max_length=64)
|
|
132
|
+
|
|
133
|
+
@model_validator(mode="after")
|
|
134
|
+
def _bounds_fit(self) -> "InferParam":
|
|
135
|
+
numeric = self.type in ("int", "float")
|
|
136
|
+
if (self.min is not None or self.max is not None) and not numeric:
|
|
137
|
+
raise ValueError(f"min / max bound a number; a {self.type} parameter takes choices instead")
|
|
138
|
+
if self.min is not None and self.max is not None and self.min > self.max:
|
|
139
|
+
raise ValueError(f"min {self.min} is above max {self.max}")
|
|
140
|
+
if self.default is not None and not allowed(self, self.default):
|
|
141
|
+
raise ValueError(f"the default {self.default!r} is outside what the parameter allows ({describe(self)})")
|
|
142
|
+
return self
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def allowed(param: "InferParam", value) -> bool:
|
|
146
|
+
"""Whether a value (already cast to the parameter's type) is one the parameter allows."""
|
|
147
|
+
if param.choices is not None and value not in param.choices:
|
|
148
|
+
return False
|
|
149
|
+
if param.min is not None and value < param.min:
|
|
150
|
+
return False
|
|
151
|
+
return not (param.max is not None and value > param.max)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def describe(param: "InferParam") -> str:
|
|
155
|
+
"""What a parameter allows, in words: "one of 5, 10", "from 1 to 10", "at least 1", "at most 10"."""
|
|
156
|
+
if param.choices is not None:
|
|
157
|
+
return "one of " + ", ".join(repr(c) for c in param.choices)
|
|
158
|
+
if param.min is not None and param.max is not None:
|
|
159
|
+
return f"from {param.min} to {param.max}"
|
|
160
|
+
if param.min is not None:
|
|
161
|
+
return f"at least {param.min}"
|
|
162
|
+
if param.max is not None:
|
|
163
|
+
return f"at most {param.max}"
|
|
164
|
+
return f"any {param.type}"
|
|
125
165
|
|
|
126
166
|
|
|
127
167
|
class DeploymentPolicy(BaseModel):
|
|
@@ -71,7 +71,9 @@ class EvalRequest(BaseModel):
|
|
|
71
71
|
max_usd: float | None = Field(default=None, gt=0)
|
|
72
72
|
# Run again only the episodes of that earlier eval that errored; the report covers both runs.
|
|
73
73
|
rerun_of: str | None = None
|
|
74
|
-
#
|
|
74
|
+
# Parameters set for every arm of this eval: by `side.name` — the policy's @seq.infer, the adapter's __init__, the
|
|
75
|
+
# benchmark's @seq.setup, or the eval loop's own (`eval.steps_per_query`) — or by the bare name when only one side
|
|
76
|
+
# declares it. `sweep` names them the same way. {gs-plans/eval-adapters/36}
|
|
75
77
|
params: dict[str, int | float | bool | str] | None = Field(default=None, max_length=32)
|
|
76
78
|
# Only the conditions whose variant takes one of these values on each named axis.
|
|
77
79
|
variants: dict[str, list[str]] | None = Field(default=None, max_length=32)
|
|
@@ -261,6 +263,8 @@ class ArmReport(BaseModel):
|
|
|
261
263
|
|
|
262
264
|
arm: int
|
|
263
265
|
model: str
|
|
266
|
+
# What this arm's eval set (the eval's params and this arm's sweep values): a policy's inference parameter by its
|
|
267
|
+
# name, any other side's as side.name (`eval.steps_per_query`). Every value it ran with is in EvalReport.arm_params.
|
|
264
268
|
params: dict[str, int | float | bool | str] = Field(default_factory=dict)
|
|
265
269
|
metrics: dict[str, MetricAgg] = Field(default_factory=dict)
|
|
266
270
|
# Each metric's 95% confidence interval as [low, high].
|
|
@@ -271,6 +275,33 @@ class ArmReport(BaseModel):
|
|
|
271
275
|
budget_stopped: bool = False
|
|
272
276
|
|
|
273
277
|
|
|
278
|
+
class ParamUsed(BaseModel):
|
|
279
|
+
"""One parameter's value in an arm, and who gave it: the eval (`cli`), the default its declaring side wrote
|
|
280
|
+
(`policy` / `adapter` / `benchmark`), or the platform's own (`platform`). {gs-plans/eval-adapters/36}"""
|
|
281
|
+
|
|
282
|
+
model_config = ConfigDict(extra="forbid")
|
|
283
|
+
|
|
284
|
+
value: int | float | bool | str | None = None
|
|
285
|
+
source: Literal["cli", "policy", "adapter", "benchmark", "platform"]
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
class ArmParams(BaseModel):
|
|
289
|
+
"""An arm's parameters, side by side: every one the policy's @seq.infer, the adapter's __init__, the benchmark's
|
|
290
|
+
@seq.setup and the eval loop declare, with the value the arm ran with and where it came from; and how many of its
|
|
291
|
+
steps asked the policy (their observation read, so rendered, once each) of all the steps it took — what
|
|
292
|
+
`eval.steps_per_query` saves. {gs-plans/eval-adapters/36}"""
|
|
293
|
+
|
|
294
|
+
model_config = ConfigDict(extra="forbid")
|
|
295
|
+
|
|
296
|
+
arm: int
|
|
297
|
+
policy: dict[str, ParamUsed] = Field(default_factory=dict)
|
|
298
|
+
adapter: dict[str, ParamUsed] = Field(default_factory=dict)
|
|
299
|
+
benchmark: dict[str, ParamUsed] = Field(default_factory=dict)
|
|
300
|
+
eval: dict[str, ParamUsed] = Field(default_factory=dict)
|
|
301
|
+
queries: int | None = None
|
|
302
|
+
steps: int | None = None
|
|
303
|
+
|
|
304
|
+
|
|
274
305
|
class PairedDiff(BaseModel):
|
|
275
306
|
"""`compare`: one metric's difference between two arms over the seeds both ran (b - a), with its interval."""
|
|
276
307
|
|
|
@@ -323,6 +354,9 @@ class EvalReport(BaseModel):
|
|
|
323
354
|
paired: list[PairedDiff] | None = None
|
|
324
355
|
timing: EvalTiming | None = None
|
|
325
356
|
stop_reason: str | None = None
|
|
357
|
+
# Per arm, every parameter's value and where it came from (gs-plans eval-adapters/36). Here and not in ArmReport:
|
|
358
|
+
# an SDK from before it drops a report key its contract does not define, but would refuse one inside an arm.
|
|
359
|
+
arm_params: list[ArmParams] | None = None
|
|
326
360
|
|
|
327
361
|
|
|
328
362
|
class AdapterKind(str, Enum):
|
{sequence_base-0.3.22 → sequence_base-0.3.24}/sequence_base/schemas/BenchmarkRegisterRequest.json
RENAMED
|
@@ -415,6 +415,108 @@
|
|
|
415
415
|
"title": "ImageStepUvPip",
|
|
416
416
|
"type": "object"
|
|
417
417
|
},
|
|
418
|
+
"InferParam": {
|
|
419
|
+
"additionalProperties": false,
|
|
420
|
+
"description": "One inference parameter the policy's `@seq.infer` declares — `def infer(self, obs, *, num_steps: int = 10)`\n— with its type and default. An /act or an eval may set it; the worker passes only declared names, cast to\nthis type, and refuses any other. {gs-plans/customization/policy.md \"四、推理行为\"}\n\n`min` / `max` (a number's bounds, inclusive) and `choices` (the values it may take) — declared as\n`Annotated[int, seq.Range(1, 10)]` / `seq.OneOf(...)` — refuse any other value before inference, saying which are\nallowed; None (left off the wire) holds it to its type alone. {gs-plans/deploy-serving/35 \"允许声明范围或可选值\"}",
|
|
421
|
+
"properties": {
|
|
422
|
+
"choices": {
|
|
423
|
+
"anyOf": [
|
|
424
|
+
{
|
|
425
|
+
"items": {
|
|
426
|
+
"anyOf": [
|
|
427
|
+
{
|
|
428
|
+
"type": "integer"
|
|
429
|
+
},
|
|
430
|
+
{
|
|
431
|
+
"type": "number"
|
|
432
|
+
},
|
|
433
|
+
{
|
|
434
|
+
"type": "boolean"
|
|
435
|
+
},
|
|
436
|
+
{
|
|
437
|
+
"type": "string"
|
|
438
|
+
}
|
|
439
|
+
]
|
|
440
|
+
},
|
|
441
|
+
"maxItems": 64,
|
|
442
|
+
"minItems": 1,
|
|
443
|
+
"type": "array"
|
|
444
|
+
},
|
|
445
|
+
{
|
|
446
|
+
"type": "null"
|
|
447
|
+
}
|
|
448
|
+
],
|
|
449
|
+
"default": null,
|
|
450
|
+
"title": "Choices"
|
|
451
|
+
},
|
|
452
|
+
"default": {
|
|
453
|
+
"anyOf": [
|
|
454
|
+
{
|
|
455
|
+
"type": "integer"
|
|
456
|
+
},
|
|
457
|
+
{
|
|
458
|
+
"type": "number"
|
|
459
|
+
},
|
|
460
|
+
{
|
|
461
|
+
"type": "boolean"
|
|
462
|
+
},
|
|
463
|
+
{
|
|
464
|
+
"type": "string"
|
|
465
|
+
},
|
|
466
|
+
{
|
|
467
|
+
"type": "null"
|
|
468
|
+
}
|
|
469
|
+
],
|
|
470
|
+
"default": null,
|
|
471
|
+
"title": "Default"
|
|
472
|
+
},
|
|
473
|
+
"max": {
|
|
474
|
+
"anyOf": [
|
|
475
|
+
{
|
|
476
|
+
"type": "integer"
|
|
477
|
+
},
|
|
478
|
+
{
|
|
479
|
+
"type": "number"
|
|
480
|
+
},
|
|
481
|
+
{
|
|
482
|
+
"type": "null"
|
|
483
|
+
}
|
|
484
|
+
],
|
|
485
|
+
"default": null,
|
|
486
|
+
"title": "Max"
|
|
487
|
+
},
|
|
488
|
+
"min": {
|
|
489
|
+
"anyOf": [
|
|
490
|
+
{
|
|
491
|
+
"type": "integer"
|
|
492
|
+
},
|
|
493
|
+
{
|
|
494
|
+
"type": "number"
|
|
495
|
+
},
|
|
496
|
+
{
|
|
497
|
+
"type": "null"
|
|
498
|
+
}
|
|
499
|
+
],
|
|
500
|
+
"default": null,
|
|
501
|
+
"title": "Min"
|
|
502
|
+
},
|
|
503
|
+
"type": {
|
|
504
|
+
"enum": [
|
|
505
|
+
"int",
|
|
506
|
+
"float",
|
|
507
|
+
"bool",
|
|
508
|
+
"str"
|
|
509
|
+
],
|
|
510
|
+
"title": "Type",
|
|
511
|
+
"type": "string"
|
|
512
|
+
}
|
|
513
|
+
},
|
|
514
|
+
"required": [
|
|
515
|
+
"type"
|
|
516
|
+
],
|
|
517
|
+
"title": "InferParam",
|
|
518
|
+
"type": "object"
|
|
519
|
+
},
|
|
418
520
|
"RenderSpec": {
|
|
419
521
|
"additionalProperties": false,
|
|
420
522
|
"description": "a.k.a. `seq.Record` — declares the mp4 the platform records (money-shot is a platform tool, not user\nglue). `camera` picks which camera stream to pull frames from; `fps` is the output frame rate. Used two\nways: as `BenchmarkSpec.render` (record the whole eval) and as `Step.record` (a per-frame hint).",
|
|
@@ -665,6 +767,22 @@
|
|
|
665
767
|
"title": "Name",
|
|
666
768
|
"type": "string"
|
|
667
769
|
},
|
|
770
|
+
"params": {
|
|
771
|
+
"anyOf": [
|
|
772
|
+
{
|
|
773
|
+
"additionalProperties": {
|
|
774
|
+
"$ref": "#/$defs/InferParam"
|
|
775
|
+
},
|
|
776
|
+
"maxProperties": 32,
|
|
777
|
+
"type": "object"
|
|
778
|
+
},
|
|
779
|
+
{
|
|
780
|
+
"type": "null"
|
|
781
|
+
}
|
|
782
|
+
],
|
|
783
|
+
"default": null,
|
|
784
|
+
"title": "Params"
|
|
785
|
+
},
|
|
668
786
|
"render": {
|
|
669
787
|
"anyOf": [
|
|
670
788
|
{
|
|
@@ -718,5 +836,5 @@
|
|
|
718
836
|
],
|
|
719
837
|
"title": "BenchmarkRegisterRequest",
|
|
720
838
|
"type": "object",
|
|
721
|
-
"x-contract-version": "0.4.
|
|
839
|
+
"x-contract-version": "0.4.21"
|
|
722
840
|
}
|
|
@@ -297,7 +297,7 @@
|
|
|
297
297
|
],
|
|
298
298
|
"title": "SeqFlags",
|
|
299
299
|
"type": "integer",
|
|
300
|
-
"x-contract-version": "0.4.
|
|
300
|
+
"x-contract-version": "0.4.21",
|
|
301
301
|
"x-flag-names": {
|
|
302
302
|
"CHECK": 65536,
|
|
303
303
|
"INFER": 16,
|
|
@@ -505,5 +505,5 @@
|
|
|
505
505
|
],
|
|
506
506
|
"title": "BenchmarkSpec",
|
|
507
507
|
"type": "object",
|
|
508
|
-
"x-contract-version": "0.4.
|
|
508
|
+
"x-contract-version": "0.4.21"
|
|
509
509
|
}
|
|
@@ -170,8 +170,38 @@
|
|
|
170
170
|
},
|
|
171
171
|
"InferParam": {
|
|
172
172
|
"additionalProperties": false,
|
|
173
|
-
"description": "One inference parameter the policy's `@seq.infer` declares — `def infer(self, obs, *, num_steps: int = 10)`\n— with its type and default. An /act or an eval may set it; the worker passes only declared names, cast to\nthis type, and refuses any other. {gs-plans/customization/policy.md \"四、推理行为\"}",
|
|
173
|
+
"description": "One inference parameter the policy's `@seq.infer` declares — `def infer(self, obs, *, num_steps: int = 10)`\n— with its type and default. An /act or an eval may set it; the worker passes only declared names, cast to\nthis type, and refuses any other. {gs-plans/customization/policy.md \"四、推理行为\"}\n\n`min` / `max` (a number's bounds, inclusive) and `choices` (the values it may take) — declared as\n`Annotated[int, seq.Range(1, 10)]` / `seq.OneOf(...)` — refuse any other value before inference, saying which are\nallowed; None (left off the wire) holds it to its type alone. {gs-plans/deploy-serving/35 \"允许声明范围或可选值\"}",
|
|
174
174
|
"properties": {
|
|
175
|
+
"choices": {
|
|
176
|
+
"anyOf": [
|
|
177
|
+
{
|
|
178
|
+
"items": {
|
|
179
|
+
"anyOf": [
|
|
180
|
+
{
|
|
181
|
+
"type": "integer"
|
|
182
|
+
},
|
|
183
|
+
{
|
|
184
|
+
"type": "number"
|
|
185
|
+
},
|
|
186
|
+
{
|
|
187
|
+
"type": "boolean"
|
|
188
|
+
},
|
|
189
|
+
{
|
|
190
|
+
"type": "string"
|
|
191
|
+
}
|
|
192
|
+
]
|
|
193
|
+
},
|
|
194
|
+
"maxItems": 64,
|
|
195
|
+
"minItems": 1,
|
|
196
|
+
"type": "array"
|
|
197
|
+
},
|
|
198
|
+
{
|
|
199
|
+
"type": "null"
|
|
200
|
+
}
|
|
201
|
+
],
|
|
202
|
+
"default": null,
|
|
203
|
+
"title": "Choices"
|
|
204
|
+
},
|
|
175
205
|
"default": {
|
|
176
206
|
"anyOf": [
|
|
177
207
|
{
|
|
@@ -193,6 +223,36 @@
|
|
|
193
223
|
"default": null,
|
|
194
224
|
"title": "Default"
|
|
195
225
|
},
|
|
226
|
+
"max": {
|
|
227
|
+
"anyOf": [
|
|
228
|
+
{
|
|
229
|
+
"type": "integer"
|
|
230
|
+
},
|
|
231
|
+
{
|
|
232
|
+
"type": "number"
|
|
233
|
+
},
|
|
234
|
+
{
|
|
235
|
+
"type": "null"
|
|
236
|
+
}
|
|
237
|
+
],
|
|
238
|
+
"default": null,
|
|
239
|
+
"title": "Max"
|
|
240
|
+
},
|
|
241
|
+
"min": {
|
|
242
|
+
"anyOf": [
|
|
243
|
+
{
|
|
244
|
+
"type": "integer"
|
|
245
|
+
},
|
|
246
|
+
{
|
|
247
|
+
"type": "number"
|
|
248
|
+
},
|
|
249
|
+
{
|
|
250
|
+
"type": "null"
|
|
251
|
+
}
|
|
252
|
+
],
|
|
253
|
+
"default": null,
|
|
254
|
+
"title": "Min"
|
|
255
|
+
},
|
|
196
256
|
"type": {
|
|
197
257
|
"enum": [
|
|
198
258
|
"int",
|
|
@@ -403,5 +463,5 @@
|
|
|
403
463
|
],
|
|
404
464
|
"title": "DeploymentPolicy",
|
|
405
465
|
"type": "object",
|
|
406
|
-
"x-contract-version": "0.4.
|
|
466
|
+
"x-contract-version": "0.4.21"
|
|
407
467
|
}
|