sequence-base 0.3.18__tar.gz → 0.3.19__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sequence_base-0.3.18 → sequence_base-0.3.19}/PKG-INFO +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/pyproject.toml +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/constants/version.py +2 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/contract/benchmarks.py +14 -4
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/contract/eval.py +67 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/contract/flags.py +10 -3
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ActRequest.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ActResponse.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ActionChunk.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ActionContract.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ActionStep.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/AdapterTransform.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/AddLocal.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/Autoscaler.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/BenchmarkConditionSpec.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/BenchmarkRegisterRequest.json +20 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/BenchmarkSpec.json +25 -5
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/BenchmarkStatusResponse.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/BootstrapResponse.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/Camera.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ClampInfo.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/CodeBundle.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/Concurrency.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/Conditions.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ConnectRequest.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ConnectResponse.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/DeploymentAccepted.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/DeploymentPolicy.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/DeploymentRequest.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/DeploymentStatusResponse.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/Embodiment.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ErrorBody.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ErrorCodes.json +1 -1
- sequence_base-0.3.19/sequence_base/schemas/Estimate.json +52 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/EvalDryRunResponse.json +48 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/EvalReport.json +215 -2
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/EvalRequest.json +40 -2
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/EvalResponse.json +40 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ImageFrame.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ImageSpec.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ImageStepAddLocal.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ImageStepApt.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ImageStepEnv.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ImageStepRun.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ImageStepUvPip.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/InferParam.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/Lease.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/LossyTransform.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/MetricAgg.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/Observation.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/ObservationContract.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/PolicyRegistration.json +10 -4
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/PolicySpec.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/Proprioception.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/RenderSpec.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/Resources.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/RolloutResult.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/SecretCreateRequest.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/SecretInfo.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/SecretRef.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/SeqFlags.json +6 -2
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/Step.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/SuiteAgg.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/TaskAgg.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/Timeouts.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/UploadTarget.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/UsageReport.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/Variant.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/VolumeAttach.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/VolumeFile.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/VolumeInfo.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/VolumeMount.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/VolumeRef.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/VolumeUploadRequest.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/VolumeUploadResponse.json +1 -1
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/WarmingError.json +1 -1
- sequence_base-0.3.18/sequence_base/schemas/Estimate.json +0 -25
- {sequence_base-0.3.18 → sequence_base-0.3.19}/.gitignore +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/README.md +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/__init__.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/auth.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/codec/__init__.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/codec/image.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/constants/__init__.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/constants/env.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/constants/headers.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/contract/__init__.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/contract/accounts.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/contract/act.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/contract/connect.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/contract/deployments.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/contract/errors.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/contract/io.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/contract/policy.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/contract/templates.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/contract/volumes.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/contract/wire.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/errors/__init__.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/errors/transport.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/obs/__init__.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/obs/log.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/obs/rid.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/presets/__init__.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/presets/robot_arm.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schema_export.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/telemetry/__init__.py +0 -0
- {sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/telemetry/events.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sequence-base
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.19
|
|
4
4
|
Summary: Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness.
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Requires-Dist: pydantic>=2.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sequence-base"
|
|
3
|
-
version = "0.3.
|
|
3
|
+
version = "0.3.19"
|
|
4
4
|
description = "Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness."
|
|
5
5
|
# 3.10, not 3.11: `seq` (which imports this) must run on ROS 2 Humble's Python 3.10, and nothing here needs 3.11.
|
|
6
6
|
# {2026-10-03 the full suite on CPython 3.10.20: "147 PASSED"; vermin --eval-annotations: "MINIMUM REQUIRED VERSIONS: 3.9"}
|
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
# across deployments / repos can tell which version of a shape it is validating against. MUST change
|
|
5
5
|
# whenever any shared wire shape changes (a field added / removed / retyped) — that is what makes
|
|
6
6
|
# contract evolution mechanically detectable instead of eyeballed. Bump on any shape change.
|
|
7
|
-
CONTRACT_VERSION = "0.4.
|
|
7
|
+
CONTRACT_VERSION = "0.4.16" # 0.4.15 -> 0.4.16: batched benchmarks — SeqFlags.OBSERVE / STEP, BenchmarkSpec / BenchmarkRegisterRequest.envs and .batched; per-arm eval reports — EvalReport.arms / paired / timing / stop_reason, EvalStatus.CANCELED, Estimate.breakdown / default_timing, EvalResponse.arms, EvalDryRunResponse.layout / notes
|
|
8
|
+
# previous: 0.4.15 # 0.4.14 -> 0.4.15: limits count GPUs and CPU/memory are billed like Modal — Resources.cpu / memory_gb optional floats + cpu_limit / memory_limit_gb; Autoscaler.max_gpus / on_full / queue_timeout_s replace max_containers; Concurrency fields optional + replicas_per_gpu / gpus_per_replica; DeploymentStatusResponse.reason; EvalRequest.policy_instances / policy_gpu / benchmark_instances / benchmark_gpu / envs replace gpus / envs_per_gpu; EvalStatus.QUEUED; EvalCards / EvalQueue
|
|
8
9
|
# previous: 0.4.14 # 0.4.13 -> 0.4.14: the /act wire is opaque (ActRequest.inputs / ActResponse.outputs); action / observation contracts optional + DeploymentPolicy/PolicySpec/ConnectResponse .io; the robot-arm types live in sequence_base.presets.robot_arm (contract.act / contract.io re-export them); ImageStepAddLocal (seq.Image.add_local_*); ActionSpace.JOINT_VELOCITY (pi0.5 DROID's joint velocities + gripper position)
|
|
9
10
|
# previous: 0.4.13 # 0.4.12 -> 0.4.13: template://<name>[@v<n>] — official template weights (parse_template); gs:// / s3:// / https:// still accepted until their removal
|
|
10
11
|
# previous: 0.4.12 # 0.4.11 -> 0.4.12: one error definition — ErrorBody is every non-2xx body (detail, code, retryable, retry_after_s, request_id, hint, details), ErrorCode covers every case, ERRORS / ErrorCodes.json give each code its status, retryability, SDK class and CLI exit code
|
|
@@ -16,7 +16,7 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
|
16
16
|
|
|
17
17
|
from sequence_base.contract.accounts import SecretRef
|
|
18
18
|
from sequence_base.contract.deployments import CodeBundle, DeploymentState
|
|
19
|
-
from sequence_base.contract.flags import BENCHMARK_BINDABLE, SeqFlags
|
|
19
|
+
from sequence_base.contract.flags import BENCHMARK_BATCHED, BENCHMARK_BINDABLE, SeqFlags
|
|
20
20
|
from sequence_base.contract.policy import ActionContract, ImageSpec, VolumeRef
|
|
21
21
|
|
|
22
22
|
__all__ = [
|
|
@@ -92,8 +92,10 @@ class BenchmarkSpec(BaseModel):
|
|
|
92
92
|
conditions: Conditions
|
|
93
93
|
max_steps: int = Field(..., description="Platform safety cap; truncates and records truncated")
|
|
94
94
|
handler_ref: str
|
|
95
|
-
bindings: dict[SeqFlags, str] = Field(..., description="SETUP / RESET / CHECK / SCORE → method name")
|
|
95
|
+
bindings: dict[SeqFlags, str] = Field(..., description="SETUP / RESET / CHECK / SCORE (+ OBSERVE / STEP) → method name")
|
|
96
96
|
gpu: str | list[str] | None = None
|
|
97
|
+
# How many environments one benchmark instance runs at once when an eval does not say (the author's default).
|
|
98
|
+
envs: int | None = Field(default=None, ge=1, le=65536)
|
|
97
99
|
assets: list[VolumeRef] = Field(default_factory=list)
|
|
98
100
|
render: RenderSpec | None = None
|
|
99
101
|
cpu: int | None = None
|
|
@@ -108,9 +110,12 @@ class BenchmarkSpec(BaseModel):
|
|
|
108
110
|
if bad:
|
|
109
111
|
names = ", ".join(sorted(f.name or str(f.value) for f in bad))
|
|
110
112
|
raise ValueError(
|
|
111
|
-
f"benchmark bindings may only cover SETUP / RESET / CHECK / SCORE; got disallowed {names}"
|
|
113
|
+
f"benchmark bindings may only cover SETUP / RESET / CHECK / SCORE / OBSERVE / STEP; got disallowed {names}"
|
|
112
114
|
" (ROLLOUT is deprecated and must not be bound)"
|
|
113
115
|
)
|
|
116
|
+
batched = set(self.bindings) & set(BENCHMARK_BATCHED)
|
|
117
|
+
if batched and batched != set(BENCHMARK_BATCHED):
|
|
118
|
+
raise ValueError("a batched benchmark binds both OBSERVE and STEP")
|
|
114
119
|
return self
|
|
115
120
|
|
|
116
121
|
|
|
@@ -166,8 +171,13 @@ class BenchmarkRegisterRequest(BaseModel):
|
|
|
166
171
|
timeout_s: float | None = Field(default=None, gt=0)
|
|
167
172
|
# The adapter (module:Class in the benchmark's own code) an eval uses when it names none.
|
|
168
173
|
default_adapter_ref: str | None = None
|
|
169
|
-
#
|
|
174
|
+
# The card each benchmark instance runs on (L4 or RTX-PRO-6000); None takes the platform's default (L4). The
|
|
175
|
+
# simulation never runs on a CPU. {gs-plans/benchmark-internals/design.md "一、规则"}
|
|
170
176
|
gpu: str | None = Field(default=None, max_length=32)
|
|
177
|
+
# How many environments one instance runs at once when an eval does not say; None takes 1.
|
|
178
|
+
envs: int | None = Field(default=None, ge=1, le=65536)
|
|
179
|
+
# Whether the benchmark is batched (binds OBSERVE and STEP and steps many environments in one call).
|
|
180
|
+
batched: bool = False
|
|
171
181
|
# The Dockerfile's build context, as for a deployment.
|
|
172
182
|
image_context: CodeBundle | None = None
|
|
173
183
|
|
|
@@ -17,6 +17,9 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
|
17
17
|
__all__ = [
|
|
18
18
|
"EvalRequest",
|
|
19
19
|
"ClampInfo",
|
|
20
|
+
"ArmReport",
|
|
21
|
+
"PairedDiff",
|
|
22
|
+
"EvalTiming",
|
|
20
23
|
"Estimate",
|
|
21
24
|
"EvalResponse",
|
|
22
25
|
"EvalDryRunResponse",
|
|
@@ -57,7 +60,10 @@ class EvalRequest(BaseModel):
|
|
|
57
60
|
envs: int | None = Field(default=None, ge=1, le=100000)
|
|
58
61
|
vlm_max_concurrency: int | None = None
|
|
59
62
|
adapter: str | None = None
|
|
60
|
-
|
|
63
|
+
# Arms run side by side in one eval: other models on the same episodes (each its own policy instances), and/or a
|
|
64
|
+
# grid of inference-parameter values (every combination is one arm).
|
|
65
|
+
compare: str | list[str] | None = Field(default=None, description="Other model ids to compare against")
|
|
66
|
+
sweep: dict[str, list[int | float | bool | str]] | None = Field(default=None, max_length=8)
|
|
61
67
|
dry_run: bool = False
|
|
62
68
|
# Only these tasks (of the selected suite / trials). {gs-plans/customization/benchmark.md "子集"}
|
|
63
69
|
tasks: list[str] | None = Field(default=None, max_length=1000)
|
|
@@ -90,6 +96,10 @@ class Estimate(BaseModel):
|
|
|
90
96
|
wall_clock_s: float
|
|
91
97
|
cost_usd: float
|
|
92
98
|
vlm_calls: int
|
|
99
|
+
# The cost split by machine and by resource (GPU / CPU / memory), when the gateway itemizes it.
|
|
100
|
+
breakdown: dict[str, float] | None = None
|
|
101
|
+
# True when no run of this benchmark version has been measured yet and the estimate uses the default step time.
|
|
102
|
+
default_timing: bool | None = None
|
|
93
103
|
|
|
94
104
|
|
|
95
105
|
class EvalResponse(BaseModel):
|
|
@@ -103,6 +113,8 @@ class EvalResponse(BaseModel):
|
|
|
103
113
|
clamped_by: ClampInfo | None = None
|
|
104
114
|
# Set when the account was short of GPUs: the eval waits (up to its timeout) instead of being refused.
|
|
105
115
|
queue: EvalQueue | None = None
|
|
116
|
+
# How many arms (model x swept parameters) the eval runs side by side.
|
|
117
|
+
arms: int | None = None
|
|
106
118
|
|
|
107
119
|
|
|
108
120
|
class CardSide(BaseModel):
|
|
@@ -160,6 +172,10 @@ class EvalDryRunResponse(BaseModel):
|
|
|
160
172
|
|
|
161
173
|
estimate: Estimate
|
|
162
174
|
cards: EvalCards | None = None
|
|
175
|
+
# The eval's layout as it would run: per side the instances, card and environments (the user's numbers or the
|
|
176
|
+
# fixed defaults), and the arms. Advice only ever comes as `notes`; the platform never changes the numbers.
|
|
177
|
+
layout: dict | None = None
|
|
178
|
+
notes: list[str] = Field(default_factory=list)
|
|
163
179
|
|
|
164
180
|
|
|
165
181
|
class RolloutResult(BaseModel):
|
|
@@ -190,6 +206,7 @@ class EvalStatus(str, Enum):
|
|
|
190
206
|
RUNNING = "running"
|
|
191
207
|
COMPLETED = "completed"
|
|
192
208
|
FAILED = "failed"
|
|
209
|
+
CANCELED = "canceled"
|
|
193
210
|
|
|
194
211
|
|
|
195
212
|
class SuiteAgg(BaseModel):
|
|
@@ -236,6 +253,49 @@ class LossyTransform(BaseModel):
|
|
|
236
253
|
reason: str
|
|
237
254
|
|
|
238
255
|
|
|
256
|
+
class ArmReport(BaseModel):
|
|
257
|
+
"""One arm of an eval (a model x one combination of swept inference parameters): its own metrics and episode
|
|
258
|
+
counts, so arms are compared side by side and never pooled."""
|
|
259
|
+
|
|
260
|
+
model_config = ConfigDict(extra="forbid")
|
|
261
|
+
|
|
262
|
+
arm: int
|
|
263
|
+
model: str
|
|
264
|
+
params: dict[str, int | float | bool | str] = Field(default_factory=dict)
|
|
265
|
+
metrics: dict[str, MetricAgg] = Field(default_factory=dict)
|
|
266
|
+
# Each metric's 95% confidence interval as [low, high].
|
|
267
|
+
intervals: dict[str, list[float]] = Field(default_factory=dict)
|
|
268
|
+
episodes: int = 0
|
|
269
|
+
failed: int = 0
|
|
270
|
+
not_run: int = 0
|
|
271
|
+
budget_stopped: bool = False
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
class PairedDiff(BaseModel):
|
|
275
|
+
"""`compare`: one metric's difference between two arms over the seeds both ran (b - a), with its interval."""
|
|
276
|
+
|
|
277
|
+
model_config = ConfigDict(extra="forbid")
|
|
278
|
+
|
|
279
|
+
metric: str
|
|
280
|
+
arm_a: int
|
|
281
|
+
arm_b: int
|
|
282
|
+
n: int
|
|
283
|
+
mean_diff: float
|
|
284
|
+
interval: list[float]
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
class EvalTiming(BaseModel):
|
|
288
|
+
"""Where an eval's time went: cold starts, and per step the simulation, rendering, the policy's batched forward and
|
|
289
|
+
the network around it, plus the forward's average batch size."""
|
|
290
|
+
|
|
291
|
+
model_config = ConfigDict(extra="forbid")
|
|
292
|
+
|
|
293
|
+
cold_start_s: dict[str, float] = Field(default_factory=dict)
|
|
294
|
+
step_ms: dict[str, float] = Field(default_factory=dict)
|
|
295
|
+
batch_size_mean: float | None = None
|
|
296
|
+
steps: int = 0
|
|
297
|
+
|
|
298
|
+
|
|
239
299
|
class EvalReport(BaseModel):
|
|
240
300
|
"""GET /v1/eval/{id} (eval-raw revision). NO hard-coded `overall_sr` / `success_rate` top-level field —
|
|
241
301
|
`metrics` is a passthrough dict keyed by the user's metric names (from `@seq.check`'s `Step.metrics`),
|
|
@@ -257,6 +317,12 @@ class EvalReport(BaseModel):
|
|
|
257
317
|
errors: list[str] = Field(
|
|
258
318
|
default_factory=list, description="Why episodes or the run failed, so an incomplete report says why"
|
|
259
319
|
)
|
|
320
|
+
# Per arm (one entry for an eval with one arm), paired comparisons for `compare`, the timing breakdown, and why
|
|
321
|
+
# the run stopped early (canceled / budget), when it did.
|
|
322
|
+
arms: list[ArmReport] | None = None
|
|
323
|
+
paired: list[PairedDiff] | None = None
|
|
324
|
+
timing: EvalTiming | None = None
|
|
325
|
+
stop_reason: str | None = None
|
|
260
326
|
|
|
261
327
|
|
|
262
328
|
class AdapterKind(str, Enum):
|
|
@@ -41,18 +41,25 @@ class SeqFlags(IntFlag):
|
|
|
41
41
|
SCORE = 16384 # cross-episode aggregate (optional; default = mean of each numeric metric)
|
|
42
42
|
RESET = 32768 # build one episode's env (gym-style handle) — eval-raw new hook, bit 15
|
|
43
43
|
CHECK = 65536 # per step: read raw env state → done + user-named metrics — eval-raw new hook, bit 16
|
|
44
|
+
# A batched benchmark (a GPU sim that steps many environments at once) also binds these two; its RESET then
|
|
45
|
+
# takes (slots, conditions) and its CHECK (slots). {gs-plans/benchmark-internals/design.md "三、两种写法"}
|
|
46
|
+
OBSERVE = 131072 # read the observation of the given slots, one per slot — bit 17
|
|
47
|
+
STEP = 262144 # apply one action to each given slot — bit 18
|
|
44
48
|
# DEPRECATED — the old single-hook benchmark model. Bit 8192 stays reserved (never reused for another
|
|
45
49
|
# member) but is NOT bound in any registration; setup/reset/check/score replaces it. Kept only so the
|
|
46
50
|
# value cannot be recycled and an old serialized 8192 still decodes to a known name.
|
|
47
51
|
ROLLOUT = 8192
|
|
48
52
|
|
|
49
53
|
|
|
50
|
-
# The benchmark lifecycle bits a live registration may bind (eval-raw): SETUP / RESET / CHECK / SCORE
|
|
51
|
-
# ROLLOUT is deliberately absent — deprecated, never bound.
|
|
54
|
+
# The benchmark lifecycle bits a live registration may bind (eval-raw): SETUP / RESET / CHECK / SCORE, and OBSERVE /
|
|
55
|
+
# STEP for a batched benchmark. ROLLOUT is deliberately absent — deprecated, never bound.
|
|
52
56
|
BENCHMARK_BINDABLE: frozenset[SeqFlags] = frozenset(
|
|
53
|
-
{SeqFlags.SETUP, SeqFlags.RESET, SeqFlags.CHECK, SeqFlags.SCORE}
|
|
57
|
+
{SeqFlags.SETUP, SeqFlags.RESET, SeqFlags.CHECK, SeqFlags.SCORE, SeqFlags.OBSERVE, SeqFlags.STEP}
|
|
54
58
|
)
|
|
55
59
|
|
|
60
|
+
# The two bits that make a benchmark batched: present together or not at all.
|
|
61
|
+
BENCHMARK_BATCHED: frozenset[SeqFlags] = frozenset({SeqFlags.OBSERVE, SeqFlags.STEP})
|
|
62
|
+
|
|
56
63
|
# Deprecated benchmark bits: reserved value, must not appear in any bindings.
|
|
57
64
|
DEPRECATED_FLAGS: frozenset[SeqFlags] = frozenset({SeqFlags.ROLLOUT})
|
|
58
65
|
|
{sequence_base-0.3.18 → sequence_base-0.3.19}/sequence_base/schemas/BenchmarkRegisterRequest.json
RENAMED
|
@@ -523,6 +523,11 @@
|
|
|
523
523
|
"title": "Assets",
|
|
524
524
|
"type": "array"
|
|
525
525
|
},
|
|
526
|
+
"batched": {
|
|
527
|
+
"default": false,
|
|
528
|
+
"title": "Batched",
|
|
529
|
+
"type": "boolean"
|
|
530
|
+
},
|
|
526
531
|
"code": {
|
|
527
532
|
"anyOf": [
|
|
528
533
|
{
|
|
@@ -577,6 +582,20 @@
|
|
|
577
582
|
"default": null,
|
|
578
583
|
"title": "Default Adapter Ref"
|
|
579
584
|
},
|
|
585
|
+
"envs": {
|
|
586
|
+
"anyOf": [
|
|
587
|
+
{
|
|
588
|
+
"maximum": 65536,
|
|
589
|
+
"minimum": 1,
|
|
590
|
+
"type": "integer"
|
|
591
|
+
},
|
|
592
|
+
{
|
|
593
|
+
"type": "null"
|
|
594
|
+
}
|
|
595
|
+
],
|
|
596
|
+
"default": null,
|
|
597
|
+
"title": "Envs"
|
|
598
|
+
},
|
|
580
599
|
"gpu": {
|
|
581
600
|
"anyOf": [
|
|
582
601
|
{
|
|
@@ -681,5 +700,5 @@
|
|
|
681
700
|
],
|
|
682
701
|
"title": "BenchmarkRegisterRequest",
|
|
683
702
|
"type": "object",
|
|
684
|
-
"x-contract-version": "0.4.
|
|
703
|
+
"x-contract-version": "0.4.16"
|
|
685
704
|
}
|
|
@@ -273,15 +273,18 @@
|
|
|
273
273
|
8192,
|
|
274
274
|
16384,
|
|
275
275
|
32768,
|
|
276
|
-
65536
|
|
276
|
+
65536,
|
|
277
|
+
131072,
|
|
278
|
+
262144
|
|
277
279
|
],
|
|
278
280
|
"title": "SeqFlags",
|
|
279
281
|
"type": "integer",
|
|
280
|
-
"x-contract-version": "0.4.
|
|
282
|
+
"x-contract-version": "0.4.16",
|
|
281
283
|
"x-flag-names": {
|
|
282
284
|
"CHECK": 65536,
|
|
283
285
|
"INFER": 16,
|
|
284
286
|
"LOAD": 2,
|
|
287
|
+
"OBSERVE": 131072,
|
|
285
288
|
"ON_EPISODE": 1024,
|
|
286
289
|
"PLAN": 4,
|
|
287
290
|
"RESET": 32768,
|
|
@@ -289,6 +292,7 @@
|
|
|
289
292
|
"SCORE": 16384,
|
|
290
293
|
"SETUP": 4096,
|
|
291
294
|
"SHUTDOWN": 8,
|
|
295
|
+
"STEP": 262144,
|
|
292
296
|
"WARMUP": 1
|
|
293
297
|
}
|
|
294
298
|
},
|
|
@@ -357,7 +361,7 @@
|
|
|
357
361
|
"additionalProperties": {
|
|
358
362
|
"type": "string"
|
|
359
363
|
},
|
|
360
|
-
"description": "SETUP / RESET / CHECK / SCORE → method name",
|
|
364
|
+
"description": "SETUP / RESET / CHECK / SCORE (+ OBSERVE / STEP) → method name",
|
|
361
365
|
"propertyNames": {
|
|
362
366
|
"enum": [
|
|
363
367
|
"1",
|
|
@@ -370,7 +374,9 @@
|
|
|
370
374
|
"8192",
|
|
371
375
|
"16384",
|
|
372
376
|
"32768",
|
|
373
|
-
"65536"
|
|
377
|
+
"65536",
|
|
378
|
+
"131072",
|
|
379
|
+
"262144"
|
|
374
380
|
],
|
|
375
381
|
"type": "string"
|
|
376
382
|
},
|
|
@@ -392,6 +398,20 @@
|
|
|
392
398
|
"default": null,
|
|
393
399
|
"title": "Cpu"
|
|
394
400
|
},
|
|
401
|
+
"envs": {
|
|
402
|
+
"anyOf": [
|
|
403
|
+
{
|
|
404
|
+
"maximum": 65536,
|
|
405
|
+
"minimum": 1,
|
|
406
|
+
"type": "integer"
|
|
407
|
+
},
|
|
408
|
+
{
|
|
409
|
+
"type": "null"
|
|
410
|
+
}
|
|
411
|
+
],
|
|
412
|
+
"default": null,
|
|
413
|
+
"title": "Envs"
|
|
414
|
+
},
|
|
395
415
|
"gpu": {
|
|
396
416
|
"anyOf": [
|
|
397
417
|
{
|
|
@@ -467,5 +487,5 @@
|
|
|
467
487
|
],
|
|
468
488
|
"title": "BenchmarkSpec",
|
|
469
489
|
"type": "object",
|
|
470
|
-
"x-contract-version": "0.4.
|
|
490
|
+
"x-contract-version": "0.4.16"
|
|
471
491
|
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
{
|
|
2
|
+
"additionalProperties": false,
|
|
3
|
+
"properties": {
|
|
4
|
+
"breakdown": {
|
|
5
|
+
"anyOf": [
|
|
6
|
+
{
|
|
7
|
+
"additionalProperties": {
|
|
8
|
+
"type": "number"
|
|
9
|
+
},
|
|
10
|
+
"type": "object"
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"type": "null"
|
|
14
|
+
}
|
|
15
|
+
],
|
|
16
|
+
"default": null,
|
|
17
|
+
"title": "Breakdown"
|
|
18
|
+
},
|
|
19
|
+
"cost_usd": {
|
|
20
|
+
"title": "Cost Usd",
|
|
21
|
+
"type": "number"
|
|
22
|
+
},
|
|
23
|
+
"default_timing": {
|
|
24
|
+
"anyOf": [
|
|
25
|
+
{
|
|
26
|
+
"type": "boolean"
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"type": "null"
|
|
30
|
+
}
|
|
31
|
+
],
|
|
32
|
+
"default": null,
|
|
33
|
+
"title": "Default Timing"
|
|
34
|
+
},
|
|
35
|
+
"vlm_calls": {
|
|
36
|
+
"title": "Vlm Calls",
|
|
37
|
+
"type": "integer"
|
|
38
|
+
},
|
|
39
|
+
"wall_clock_s": {
|
|
40
|
+
"title": "Wall Clock S",
|
|
41
|
+
"type": "number"
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
"required": [
|
|
45
|
+
"wall_clock_s",
|
|
46
|
+
"cost_usd",
|
|
47
|
+
"vlm_calls"
|
|
48
|
+
],
|
|
49
|
+
"title": "Estimate",
|
|
50
|
+
"type": "object",
|
|
51
|
+
"x-contract-version": "0.4.16"
|
|
52
|
+
}
|