sequence-base 0.3.22__tar.gz → 0.3.23__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sequence_base-0.3.22 → sequence_base-0.3.23}/PKG-INFO +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/pyproject.toml +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/constants/version.py +2 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/contract/benchmarks.py +12 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/contract/eval.py +35 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ActRequest.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ActResponse.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ActionChunk.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ActionContract.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ActionStep.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/AdapterTransform.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/AddLocal.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/Autoscaler.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/BenchmarkConditionSpec.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/BenchmarkRegisterRequest.json +59 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/BenchmarkSpec.json +2 -2
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/BenchmarkStatusResponse.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/BootstrapResponse.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/Camera.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ClampInfo.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/CodeBundle.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/Concurrency.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/Conditions.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ConnectRequest.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ConnectResponse.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/DeploymentAccepted.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/DeploymentPolicy.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/DeploymentRequest.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/DeploymentStatusResponse.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/Embodiment.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ErrorBody.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ErrorCodes.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/Estimate.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/EvalDryRunResponse.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/EvalReport.json +126 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/EvalRequest.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/EvalResponse.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ImageFrame.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ImageSpec.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ImageStepAddLocal.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ImageStepApt.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ImageStepEnv.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ImageStepRun.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ImageStepUvPip.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/InferParam.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/Lease.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/LossyTransform.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/MetricAgg.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/Observation.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/ObservationContract.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/PolicyRegistration.json +2 -2
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/PolicySpec.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/Proprioception.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/RenderSpec.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/Resources.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/RolloutResult.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/SecretCreateRequest.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/SecretInfo.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/SecretRef.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/SeqFlags.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/Step.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/SuiteAgg.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/TaskAgg.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/Timeouts.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/UploadTarget.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/UsageReport.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/Variant.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/VolumeAttach.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/VolumeFile.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/VolumeInfo.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/VolumeMount.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/VolumeRef.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/VolumeUploadRequest.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/VolumeUploadResponse.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/WarmingError.json +1 -1
- {sequence_base-0.3.22 → sequence_base-0.3.23}/.gitignore +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/README.md +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/__init__.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/auth.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/codec/__init__.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/codec/image.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/constants/__init__.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/constants/env.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/constants/headers.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/contract/__init__.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/contract/accounts.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/contract/act.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/contract/connect.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/contract/deployments.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/contract/errors.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/contract/flags.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/contract/io.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/contract/policy.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/contract/templates.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/contract/volumes.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/contract/wire.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/errors/__init__.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/errors/transport.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/obs/__init__.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/obs/log.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/obs/rid.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/presets/__init__.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/presets/robot_arm.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schema_export.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/telemetry/__init__.py +0 -0
- {sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/telemetry/events.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sequence-base
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.23
|
|
4
4
|
Summary: Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness.
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Requires-Dist: pydantic>=2.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sequence-base"
|
|
3
|
-
version = "0.3.
|
|
3
|
+
version = "0.3.23"
|
|
4
4
|
description = "Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness."
|
|
5
5
|
# 3.10, not 3.11: `seq` (which imports this) must run on ROS 2 Humble's Python 3.10, and nothing here needs 3.11.
|
|
6
6
|
# {2026-10-03 the full suite on CPython 3.10.20: "147 PASSED"; vermin --eval-annotations: "MINIMUM REQUIRED VERSIONS: 3.9"}
|
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
# across deployments / repos can tell which version of a shape it is validating against. MUST change
|
|
5
5
|
# whenever any shared wire shape changes (a field added / removed / retyped) — that is what makes
|
|
6
6
|
# contract evolution mechanically detectable instead of eyeballed. Bump on any shape change.
|
|
7
|
-
CONTRACT_VERSION = "0.4.
|
|
7
|
+
CONTRACT_VERSION = "0.4.20" # 0.4.19 -> 0.4.20: BenchmarkRegisterRequest.params — the parameters a benchmark's @seq.setup declares (InferParam: type and default), which an eval may set; EvalReport.arm_params — per arm each parameter's value and source, and the steps that asked the policy of all it took (gs-plans eval-adapters/36)
|
|
8
|
+
# previous: 0.4.19 # 0.4.18 -> 0.4.19: RenderSpec.flip — how the platform turns a recorded camera's frames before encoding ("vertical" / "horizontal" / "both"; None, left off the wire, as drawn), declared once by the benchmark's author (seq.Record(flip=...)); gs-plans benchmarks
|
|
8
9
|
# previous: 0.4.18 # 0.4.17 -> 0.4.18: DeploymentPolicy.models — the platform models a policy calls while it runs (catalogue ids), which the act key driving it may call on /v1/chat and /v1/decide (gs-plans models-api/11)
|
|
9
10
|
# previous: 0.4.17 # 0.4.16 -> 0.4.17: weights optional — PolicySpec.weights, DeploymentRequest.weights and BootstrapResponse.weights_uri may be None, for a policy without a checkpoint (nothing copied or mounted; gs-plans storage T13)
|
|
10
11
|
# previous: 0.4.16 # 0.4.15 -> 0.4.16: batched benchmarks — SeqFlags.OBSERVE / STEP, BenchmarkSpec / BenchmarkRegisterRequest.envs and .batched; per-arm eval reports — EvalReport.arms / paired / timing / stop_reason, EvalStatus.CANCELED, Estimate.breakdown / default_timing, EvalResponse.arms, EvalDryRunResponse.layout / notes
|
|
@@ -17,7 +17,7 @@ from typing import Literal
|
|
|
17
17
|
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
18
18
|
|
|
19
19
|
from sequence_base.contract.accounts import SecretRef
|
|
20
|
-
from sequence_base.contract.deployments import CodeBundle, DeploymentState
|
|
20
|
+
from sequence_base.contract.deployments import CodeBundle, DeploymentState, InferParam
|
|
21
21
|
from sequence_base.contract.flags import BENCHMARK_BATCHED, BENCHMARK_BINDABLE, SeqFlags
|
|
22
22
|
from sequence_base.contract.policy import ActionContract, ImageSpec, VolumeRef
|
|
23
23
|
|
|
@@ -185,11 +185,22 @@ class BenchmarkRegisterRequest(BaseModel):
|
|
|
185
185
|
gpu: str | None = Field(default=None, max_length=32)
|
|
186
186
|
# How many environments one instance runs at once when an eval does not say; None takes 1.
|
|
187
187
|
envs: int | None = Field(default=None, ge=1, le=65536)
|
|
188
|
+
# The parameters its @seq.setup declares — `def setup(self, envs, *, noise_std: float = 0.02)` — each with its type
|
|
189
|
+
# and default, which an eval may set (--param benchmark.noise_std=0); None: it declares none, or an SDK from before
|
|
190
|
+
# them. {gs-plans/eval-adapters/36}
|
|
191
|
+
params: dict[str, InferParam] | None = Field(default=None, max_length=32)
|
|
188
192
|
# Whether the benchmark is batched (binds OBSERVE and STEP and steps many environments in one call).
|
|
189
193
|
batched: bool = False
|
|
190
194
|
# The Dockerfile's build context, as for a deployment.
|
|
191
195
|
image_context: CodeBundle | None = None
|
|
192
196
|
|
|
197
|
+
@model_validator(mode="after")
|
|
198
|
+
def _param_names(self) -> "BenchmarkRegisterRequest":
|
|
199
|
+
for name in self.params or {}:
|
|
200
|
+
if not name.isidentifier() or name.startswith("_"):
|
|
201
|
+
raise ValueError(f"setup parameter {name!r} is not a public Python identifier")
|
|
202
|
+
return self
|
|
203
|
+
|
|
193
204
|
|
|
194
205
|
class BenchmarkStatusResponse(BaseModel):
|
|
195
206
|
"""GET /v1/benchmarks/{name} — the build state of a registered benchmark (same lifecycle as a deployment:
|
|
@@ -71,7 +71,9 @@ class EvalRequest(BaseModel):
|
|
|
71
71
|
max_usd: float | None = Field(default=None, gt=0)
|
|
72
72
|
# Run again only the episodes of that earlier eval that errored; the report covers both runs.
|
|
73
73
|
rerun_of: str | None = None
|
|
74
|
-
#
|
|
74
|
+
# Parameters set for every arm of this eval: by `side.name` — the policy's @seq.infer, the adapter's __init__, the
|
|
75
|
+
# benchmark's @seq.setup, or the eval loop's own (`eval.steps_per_query`) — or by the bare name when only one side
|
|
76
|
+
# declares it. `sweep` names them the same way. {gs-plans/eval-adapters/36}
|
|
75
77
|
params: dict[str, int | float | bool | str] | None = Field(default=None, max_length=32)
|
|
76
78
|
# Only the conditions whose variant takes one of these values on each named axis.
|
|
77
79
|
variants: dict[str, list[str]] | None = Field(default=None, max_length=32)
|
|
@@ -261,6 +263,8 @@ class ArmReport(BaseModel):
|
|
|
261
263
|
|
|
262
264
|
arm: int
|
|
263
265
|
model: str
|
|
266
|
+
# What this arm's eval set (the eval's params and this arm's sweep values): a policy's inference parameter by its
|
|
267
|
+
# name, any other side's as side.name (`eval.steps_per_query`). Every value it ran with is in EvalReport.arm_params.
|
|
264
268
|
params: dict[str, int | float | bool | str] = Field(default_factory=dict)
|
|
265
269
|
metrics: dict[str, MetricAgg] = Field(default_factory=dict)
|
|
266
270
|
# Each metric's 95% confidence interval as [low, high].
|
|
@@ -271,6 +275,33 @@ class ArmReport(BaseModel):
|
|
|
271
275
|
budget_stopped: bool = False
|
|
272
276
|
|
|
273
277
|
|
|
278
|
+
class ParamUsed(BaseModel):
|
|
279
|
+
"""One parameter's value in an arm, and who gave it: the eval (`cli`), the default its declaring side wrote
|
|
280
|
+
(`policy` / `adapter` / `benchmark`), or the platform's own (`platform`). {gs-plans/eval-adapters/36}"""
|
|
281
|
+
|
|
282
|
+
model_config = ConfigDict(extra="forbid")
|
|
283
|
+
|
|
284
|
+
value: int | float | bool | str | None = None
|
|
285
|
+
source: Literal["cli", "policy", "adapter", "benchmark", "platform"]
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
class ArmParams(BaseModel):
|
|
289
|
+
"""An arm's parameters, side by side: every one the policy's @seq.infer, the adapter's __init__, the benchmark's
|
|
290
|
+
@seq.setup and the eval loop declare, with the value the arm ran with and where it came from; and how many of its
|
|
291
|
+
steps asked the policy (their observation read, so rendered, once each) of all the steps it took — what
|
|
292
|
+
`eval.steps_per_query` saves. {gs-plans/eval-adapters/36}"""
|
|
293
|
+
|
|
294
|
+
model_config = ConfigDict(extra="forbid")
|
|
295
|
+
|
|
296
|
+
arm: int
|
|
297
|
+
policy: dict[str, ParamUsed] = Field(default_factory=dict)
|
|
298
|
+
adapter: dict[str, ParamUsed] = Field(default_factory=dict)
|
|
299
|
+
benchmark: dict[str, ParamUsed] = Field(default_factory=dict)
|
|
300
|
+
eval: dict[str, ParamUsed] = Field(default_factory=dict)
|
|
301
|
+
queries: int | None = None
|
|
302
|
+
steps: int | None = None
|
|
303
|
+
|
|
304
|
+
|
|
274
305
|
class PairedDiff(BaseModel):
|
|
275
306
|
"""`compare`: one metric's difference between two arms over the seeds both ran (b - a), with its interval."""
|
|
276
307
|
|
|
@@ -323,6 +354,9 @@ class EvalReport(BaseModel):
|
|
|
323
354
|
paired: list[PairedDiff] | None = None
|
|
324
355
|
timing: EvalTiming | None = None
|
|
325
356
|
stop_reason: str | None = None
|
|
357
|
+
# Per arm, every parameter's value and where it came from (gs-plans eval-adapters/36). Here and not in ArmReport:
|
|
358
|
+
# an SDK from before it drops a report key its contract does not define, but would refuse one inside an arm.
|
|
359
|
+
arm_params: list[ArmParams] | None = None
|
|
326
360
|
|
|
327
361
|
|
|
328
362
|
class AdapterKind(str, Enum):
|
{sequence_base-0.3.22 → sequence_base-0.3.23}/sequence_base/schemas/BenchmarkRegisterRequest.json
RENAMED
|
@@ -415,6 +415,48 @@
|
|
|
415
415
|
"title": "ImageStepUvPip",
|
|
416
416
|
"type": "object"
|
|
417
417
|
},
|
|
418
|
+
"InferParam": {
|
|
419
|
+
"additionalProperties": false,
|
|
420
|
+
"description": "One inference parameter the policy's `@seq.infer` declares — `def infer(self, obs, *, num_steps: int = 10)`\n— with its type and default. An /act or an eval may set it; the worker passes only declared names, cast to\nthis type, and refuses any other. {gs-plans/customization/policy.md \"四、推理行为\"}",
|
|
421
|
+
"properties": {
|
|
422
|
+
"default": {
|
|
423
|
+
"anyOf": [
|
|
424
|
+
{
|
|
425
|
+
"type": "integer"
|
|
426
|
+
},
|
|
427
|
+
{
|
|
428
|
+
"type": "number"
|
|
429
|
+
},
|
|
430
|
+
{
|
|
431
|
+
"type": "boolean"
|
|
432
|
+
},
|
|
433
|
+
{
|
|
434
|
+
"type": "string"
|
|
435
|
+
},
|
|
436
|
+
{
|
|
437
|
+
"type": "null"
|
|
438
|
+
}
|
|
439
|
+
],
|
|
440
|
+
"default": null,
|
|
441
|
+
"title": "Default"
|
|
442
|
+
},
|
|
443
|
+
"type": {
|
|
444
|
+
"enum": [
|
|
445
|
+
"int",
|
|
446
|
+
"float",
|
|
447
|
+
"bool",
|
|
448
|
+
"str"
|
|
449
|
+
],
|
|
450
|
+
"title": "Type",
|
|
451
|
+
"type": "string"
|
|
452
|
+
}
|
|
453
|
+
},
|
|
454
|
+
"required": [
|
|
455
|
+
"type"
|
|
456
|
+
],
|
|
457
|
+
"title": "InferParam",
|
|
458
|
+
"type": "object"
|
|
459
|
+
},
|
|
418
460
|
"RenderSpec": {
|
|
419
461
|
"additionalProperties": false,
|
|
420
462
|
"description": "a.k.a. `seq.Record` — declares the mp4 the platform records (money-shot is a platform tool, not user\nglue). `camera` picks which camera stream to pull frames from; `fps` is the output frame rate. Used two\nways: as `BenchmarkSpec.render` (record the whole eval) and as `Step.record` (a per-frame hint).",
|
|
@@ -665,6 +707,22 @@
|
|
|
665
707
|
"title": "Name",
|
|
666
708
|
"type": "string"
|
|
667
709
|
},
|
|
710
|
+
"params": {
|
|
711
|
+
"anyOf": [
|
|
712
|
+
{
|
|
713
|
+
"additionalProperties": {
|
|
714
|
+
"$ref": "#/$defs/InferParam"
|
|
715
|
+
},
|
|
716
|
+
"maxProperties": 32,
|
|
717
|
+
"type": "object"
|
|
718
|
+
},
|
|
719
|
+
{
|
|
720
|
+
"type": "null"
|
|
721
|
+
}
|
|
722
|
+
],
|
|
723
|
+
"default": null,
|
|
724
|
+
"title": "Params"
|
|
725
|
+
},
|
|
668
726
|
"render": {
|
|
669
727
|
"anyOf": [
|
|
670
728
|
{
|
|
@@ -718,5 +776,5 @@
|
|
|
718
776
|
],
|
|
719
777
|
"title": "BenchmarkRegisterRequest",
|
|
720
778
|
"type": "object",
|
|
721
|
-
"x-contract-version": "0.4.
|
|
779
|
+
"x-contract-version": "0.4.20"
|
|
722
780
|
}
|
|
@@ -297,7 +297,7 @@
|
|
|
297
297
|
],
|
|
298
298
|
"title": "SeqFlags",
|
|
299
299
|
"type": "integer",
|
|
300
|
-
"x-contract-version": "0.4.
|
|
300
|
+
"x-contract-version": "0.4.20",
|
|
301
301
|
"x-flag-names": {
|
|
302
302
|
"CHECK": 65536,
|
|
303
303
|
"INFER": 16,
|
|
@@ -505,5 +505,5 @@
|
|
|
505
505
|
],
|
|
506
506
|
"title": "BenchmarkSpec",
|
|
507
507
|
"type": "object",
|
|
508
|
-
"x-contract-version": "0.4.
|
|
508
|
+
"x-contract-version": "0.4.20"
|
|
509
509
|
}
|
|
@@ -1,5 +1,72 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$defs": {
|
|
3
|
+
"ArmParams": {
|
|
4
|
+
"additionalProperties": false,
|
|
5
|
+
"description": "An arm's parameters, side by side: every one the policy's @seq.infer, the adapter's __init__, the benchmark's\n@seq.setup and the eval loop declare, with the value the arm ran with and where it came from; and how many of its\nsteps asked the policy (their observation read, so rendered, once each) of all the steps it took — what\n`eval.steps_per_query` saves. {gs-plans/eval-adapters/36}",
|
|
6
|
+
"properties": {
|
|
7
|
+
"adapter": {
|
|
8
|
+
"additionalProperties": {
|
|
9
|
+
"$ref": "#/$defs/ParamUsed"
|
|
10
|
+
},
|
|
11
|
+
"title": "Adapter",
|
|
12
|
+
"type": "object"
|
|
13
|
+
},
|
|
14
|
+
"arm": {
|
|
15
|
+
"title": "Arm",
|
|
16
|
+
"type": "integer"
|
|
17
|
+
},
|
|
18
|
+
"benchmark": {
|
|
19
|
+
"additionalProperties": {
|
|
20
|
+
"$ref": "#/$defs/ParamUsed"
|
|
21
|
+
},
|
|
22
|
+
"title": "Benchmark",
|
|
23
|
+
"type": "object"
|
|
24
|
+
},
|
|
25
|
+
"eval": {
|
|
26
|
+
"additionalProperties": {
|
|
27
|
+
"$ref": "#/$defs/ParamUsed"
|
|
28
|
+
},
|
|
29
|
+
"title": "Eval",
|
|
30
|
+
"type": "object"
|
|
31
|
+
},
|
|
32
|
+
"policy": {
|
|
33
|
+
"additionalProperties": {
|
|
34
|
+
"$ref": "#/$defs/ParamUsed"
|
|
35
|
+
},
|
|
36
|
+
"title": "Policy",
|
|
37
|
+
"type": "object"
|
|
38
|
+
},
|
|
39
|
+
"queries": {
|
|
40
|
+
"anyOf": [
|
|
41
|
+
{
|
|
42
|
+
"type": "integer"
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"type": "null"
|
|
46
|
+
}
|
|
47
|
+
],
|
|
48
|
+
"default": null,
|
|
49
|
+
"title": "Queries"
|
|
50
|
+
},
|
|
51
|
+
"steps": {
|
|
52
|
+
"anyOf": [
|
|
53
|
+
{
|
|
54
|
+
"type": "integer"
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
"type": "null"
|
|
58
|
+
}
|
|
59
|
+
],
|
|
60
|
+
"default": null,
|
|
61
|
+
"title": "Steps"
|
|
62
|
+
}
|
|
63
|
+
},
|
|
64
|
+
"required": [
|
|
65
|
+
"arm"
|
|
66
|
+
],
|
|
67
|
+
"title": "ArmParams",
|
|
68
|
+
"type": "object"
|
|
69
|
+
},
|
|
3
70
|
"ArmReport": {
|
|
4
71
|
"additionalProperties": false,
|
|
5
72
|
"description": "One arm of an eval (a model x one combination of swept inference parameters): its own metrics and episode\ncounts, so arms are compared side by side and never pooled.",
|
|
@@ -226,6 +293,49 @@
|
|
|
226
293
|
"title": "PairedDiff",
|
|
227
294
|
"type": "object"
|
|
228
295
|
},
|
|
296
|
+
"ParamUsed": {
|
|
297
|
+
"additionalProperties": false,
|
|
298
|
+
"description": "One parameter's value in an arm, and who gave it: the eval (`cli`), the default its declaring side wrote\n(`policy` / `adapter` / `benchmark`), or the platform's own (`platform`). {gs-plans/eval-adapters/36}",
|
|
299
|
+
"properties": {
|
|
300
|
+
"source": {
|
|
301
|
+
"enum": [
|
|
302
|
+
"cli",
|
|
303
|
+
"policy",
|
|
304
|
+
"adapter",
|
|
305
|
+
"benchmark",
|
|
306
|
+
"platform"
|
|
307
|
+
],
|
|
308
|
+
"title": "Source",
|
|
309
|
+
"type": "string"
|
|
310
|
+
},
|
|
311
|
+
"value": {
|
|
312
|
+
"anyOf": [
|
|
313
|
+
{
|
|
314
|
+
"type": "integer"
|
|
315
|
+
},
|
|
316
|
+
{
|
|
317
|
+
"type": "number"
|
|
318
|
+
},
|
|
319
|
+
{
|
|
320
|
+
"type": "boolean"
|
|
321
|
+
},
|
|
322
|
+
{
|
|
323
|
+
"type": "string"
|
|
324
|
+
},
|
|
325
|
+
{
|
|
326
|
+
"type": "null"
|
|
327
|
+
}
|
|
328
|
+
],
|
|
329
|
+
"default": null,
|
|
330
|
+
"title": "Value"
|
|
331
|
+
}
|
|
332
|
+
},
|
|
333
|
+
"required": [
|
|
334
|
+
"source"
|
|
335
|
+
],
|
|
336
|
+
"title": "ParamUsed",
|
|
337
|
+
"type": "object"
|
|
338
|
+
},
|
|
229
339
|
"SuiteAgg": {
|
|
230
340
|
"additionalProperties": false,
|
|
231
341
|
"description": "Per-suite aggregate of ONE user-named metric — structured detail, not a bare dict. `value` is the\nmetric aggregated over that suite; the platform does not name or interpret the metric.",
|
|
@@ -287,6 +397,21 @@
|
|
|
287
397
|
"title": "Adapter Lossy",
|
|
288
398
|
"type": "array"
|
|
289
399
|
},
|
|
400
|
+
"arm_params": {
|
|
401
|
+
"anyOf": [
|
|
402
|
+
{
|
|
403
|
+
"items": {
|
|
404
|
+
"$ref": "#/$defs/ArmParams"
|
|
405
|
+
},
|
|
406
|
+
"type": "array"
|
|
407
|
+
},
|
|
408
|
+
{
|
|
409
|
+
"type": "null"
|
|
410
|
+
}
|
|
411
|
+
],
|
|
412
|
+
"default": null,
|
|
413
|
+
"title": "Arm Params"
|
|
414
|
+
},
|
|
290
415
|
"arms": {
|
|
291
416
|
"anyOf": [
|
|
292
417
|
{
|
|
@@ -397,5 +522,5 @@
|
|
|
397
522
|
],
|
|
398
523
|
"title": "EvalReport",
|
|
399
524
|
"type": "object",
|
|
400
|
-
"x-contract-version": "0.4.
|
|
525
|
+
"x-contract-version": "0.4.20"
|
|
401
526
|
}
|
|
@@ -734,7 +734,7 @@
|
|
|
734
734
|
],
|
|
735
735
|
"title": "SeqFlags",
|
|
736
736
|
"type": "integer",
|
|
737
|
-
"x-contract-version": "0.4.
|
|
737
|
+
"x-contract-version": "0.4.20",
|
|
738
738
|
"x-flag-names": {
|
|
739
739
|
"CHECK": 65536,
|
|
740
740
|
"INFER": 16,
|
|
@@ -838,5 +838,5 @@
|
|
|
838
838
|
],
|
|
839
839
|
"title": "PolicyRegistration",
|
|
840
840
|
"type": "object",
|
|
841
|
-
"x-contract-version": "0.4.
|
|
841
|
+
"x-contract-version": "0.4.20"
|
|
842
842
|
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|