sequence-base 0.4.10__tar.gz → 0.4.12__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sequence_base-0.4.10 → sequence_base-0.4.12}/PKG-INFO +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/pyproject.toml +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/__init__.py +1 -2
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/constants/version.py +2 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/contract/__init__.py +1 -2
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/contract/benchmarks.py +0 -7
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/contract/deployments.py +6 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/contract/eval.py +10 -40
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schema_export.py +0 -2
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ActRequest.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ActResponse.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ActionChunk.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ActionContract.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ActionStep.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/AdapterTransform.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/AddLocal.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/Autoscaler.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/BenchmarkConditionSpec.json +1 -13
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/BenchmarkRegisterRequest.json +1 -20
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/BenchmarkSpec.json +2 -2
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/BenchmarkStatusResponse.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/BootstrapResponse.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/Camera.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ClampInfo.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/CodeBundle.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/Concurrency.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/Conditions.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ConnectRequest.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ConnectResponse.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/DeploymentAccepted.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/DeploymentPolicy.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/DeploymentRequest.json +3 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/DeploymentStatusResponse.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/Embodiment.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ErrorBody.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ErrorCodes.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/Estimate.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/EvalDryRunResponse.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/EvalReport.json +2 -102
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/EvalRequest.json +4 -17
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/EvalResponse.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ImageFrame.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ImageSpec.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ImageStepAddLocal.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ImageStepApt.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ImageStepEnv.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ImageStepRun.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ImageStepTemplateFiles.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ImageStepUvPip.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/InferParam.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/Lease.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/LossyTransform.json +1 -1
- sequence_base-0.4.12/sequence_base/schemas/MetricAgg.json +55 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/Observation.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ObservationContract.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/PolicyRegistration.json +2 -2
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/PolicySpec.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/Proprioception.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/RenderSpec.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/ReplaySpec.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/Resources.json +3 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/RolloutResult.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/SecretCreateRequest.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/SecretInfo.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/SecretRef.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/SeqFlags.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/Step.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/TaskAgg.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/TemplateFiles.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/Timeouts.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/UploadTarget.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/Variant.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/VolumeAttach.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/VolumeFile.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/VolumeInfo.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/VolumeMount.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/VolumeRef.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/VolumeUploadRequest.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/VolumeUploadResponse.json +1 -1
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/WarmingError.json +1 -1
- sequence_base-0.4.10/sequence_base/schemas/MetricAgg.json +0 -87
- sequence_base-0.4.10/sequence_base/schemas/SuiteAgg.json +0 -26
- {sequence_base-0.4.10 → sequence_base-0.4.12}/.gitignore +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/README.md +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/auth.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/codec/__init__.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/codec/image.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/constants/__init__.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/constants/headers.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/constants/limits.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/contract/accounts.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/contract/act.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/contract/connect.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/contract/errors.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/contract/flags.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/contract/io.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/contract/policy.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/contract/templates.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/contract/volumes.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/contract/wire.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/errors/__init__.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/errors/transport.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/obs/__init__.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/obs/log.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/obs/rid.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/presets/__init__.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/presets/frames.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/presets/robot_arm.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/telemetry/__init__.py +0 -0
- {sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/telemetry/events.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sequence-base
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.12
|
|
4
4
|
Summary: Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness.
|
|
5
5
|
Project-URL: Changelog, https://generalsequences.com/docs/changelog/
|
|
6
6
|
Requires-Python: >=3.10
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sequence-base"
|
|
3
|
-
version = "0.4.
|
|
3
|
+
version = "0.4.12"
|
|
4
4
|
description = "Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness."
|
|
5
5
|
requires-python = ">=3.10"
|
|
6
6
|
dependencies = ["pydantic>=2.0"]
|
|
@@ -90,7 +90,6 @@ from sequence_base.contract import (
|
|
|
90
90
|
SecretRef,
|
|
91
91
|
SeqFlags,
|
|
92
92
|
Step,
|
|
93
|
-
SuiteAgg,
|
|
94
93
|
TaskAgg,
|
|
95
94
|
Timeouts,
|
|
96
95
|
Variant,
|
|
@@ -146,7 +145,7 @@ __all__ = [
|
|
|
146
145
|
|
|
147
146
|
"Variant", "Conditions", "RenderSpec", "ReplaySpec", "Step", "BenchmarkSpec", "BenchmarkConditionSpec", "BenchmarkRegisterRequest", "BenchmarkStatusResponse",
|
|
148
147
|
"EvalRequest", "ClampInfo", "Estimate", "EvalResponse", "EvalDryRunResponse", "RolloutResult",
|
|
149
|
-
"EvalStatus", "
|
|
148
|
+
"EvalStatus", "TaskAgg", "MetricAgg", "LossyTransform", "EvalReport",
|
|
150
149
|
"AdapterKind", "AdapterTransform",
|
|
151
150
|
|
|
152
151
|
"SecretRef", "SecretCreateRequest", "SecretInfo", "BootstrapResponse",
|
|
@@ -84,7 +84,6 @@ from sequence_base.contract.eval import (
|
|
|
84
84
|
LossyTransform,
|
|
85
85
|
MetricAgg,
|
|
86
86
|
RolloutResult,
|
|
87
|
-
SuiteAgg,
|
|
88
87
|
TaskAgg,
|
|
89
88
|
)
|
|
90
89
|
|
|
@@ -134,7 +133,7 @@ __all__ = [
|
|
|
134
133
|
|
|
135
134
|
"Variant", "Conditions", "RenderSpec", "ReplaySpec", "Step", "BenchmarkSpec", "BenchmarkConditionSpec", "BenchmarkRegisterRequest", "BenchmarkStatusResponse",
|
|
136
135
|
"EvalRequest", "ClampInfo", "Estimate", "EvalResponse", "EvalDryRunResponse", "RolloutResult",
|
|
137
|
-
"EvalStatus", "
|
|
136
|
+
"EvalStatus", "TaskAgg", "MetricAgg", "LossyTransform", "EvalReport",
|
|
138
137
|
"AdapterKind", "AdapterTransform",
|
|
139
138
|
|
|
140
139
|
"SecretRef", "SecretCreateRequest", "SecretInfo", "BootstrapResponse",
|
|
@@ -167,10 +167,6 @@ class BenchmarkConditionSpec(BaseModel):
|
|
|
167
167
|
timeout_s: float | None = Field(default=None, gt=0)
|
|
168
168
|
|
|
169
169
|
|
|
170
|
-
|
|
171
|
-
suite: str | None = None
|
|
172
|
-
|
|
173
|
-
|
|
174
170
|
class BenchmarkRegisterRequest(BaseModel):
|
|
175
171
|
"""POST /v1/benchmarks — register body, same types as the spec (ImageSpec / Conditions / VolumeRef /
|
|
176
172
|
RenderSpec). Carries assets / render / cpu / memory_gb / max_steps so the gateway can build the image,
|
|
@@ -194,9 +190,6 @@ class BenchmarkRegisterRequest(BaseModel):
|
|
|
194
190
|
code: CodeBundle | None = None
|
|
195
191
|
|
|
196
192
|
|
|
197
|
-
suite_of: dict[str, str] = Field(default_factory=dict)
|
|
198
|
-
|
|
199
|
-
|
|
200
193
|
|
|
201
194
|
task_descriptions: dict[str, str] = Field(default_factory=dict)
|
|
202
195
|
|
|
@@ -60,7 +60,12 @@ class Resources(BaseModel):
|
|
|
60
60
|
memory_limit_gb: float | None = Field(default=None, ge=0.125, le=1024)
|
|
61
61
|
|
|
62
62
|
|
|
63
|
-
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
region: str | None = Field(default=None, max_length=32, json_schema_extra={"deprecated": True},
|
|
67
|
+
description="Not accepted: a deployment's workers run wherever there are cards, so a "
|
|
68
|
+
"region is refused. Omit it (an older SDK still sends it).")
|
|
64
69
|
|
|
65
70
|
@model_validator(mode="after")
|
|
66
71
|
def _limits_cover_requests(self) -> "Resources":
|
|
@@ -20,7 +20,6 @@ __all__ = [
|
|
|
20
20
|
"EvalRequest",
|
|
21
21
|
"ClampInfo",
|
|
22
22
|
"ArmReport",
|
|
23
|
-
"PairedDiff",
|
|
24
23
|
"EvalTiming",
|
|
25
24
|
"Estimate",
|
|
26
25
|
"EvalResponse",
|
|
@@ -31,7 +30,6 @@ __all__ = [
|
|
|
31
30
|
"EvalQueue",
|
|
32
31
|
"RolloutResult",
|
|
33
32
|
"EvalStatus",
|
|
34
|
-
"SuiteAgg",
|
|
35
33
|
"TaskAgg",
|
|
36
34
|
"MetricAgg",
|
|
37
35
|
"LossyTransform",
|
|
@@ -51,7 +49,7 @@ class EvalRequest(BaseModel):
|
|
|
51
49
|
leave them out and take the earlier eval's. `dry_run` selects the estimate-only response shape below.
|
|
52
50
|
|
|
53
51
|
The one body the platform accepts: the current knobs plus the legacy names an older SDK still sends — `gpus` /
|
|
54
|
-
`envs_per_gpu` for the instance counts, `
|
|
52
|
+
`envs_per_gpu` for the instance counts, `region` — each taken and mapped, or refused with the way on,
|
|
55
53
|
never silently dropped. The gateway inherits this model rather than keeping its own."""
|
|
56
54
|
|
|
57
55
|
|
|
@@ -63,11 +61,6 @@ class EvalRequest(BaseModel):
|
|
|
63
61
|
benchmark: str | None = Field(default=None, max_length=200)
|
|
64
62
|
|
|
65
63
|
|
|
66
|
-
suite: str | None = Field(default=None, max_length=200, json_schema_extra={"deprecated": True},
|
|
67
|
-
description="Gone: a benchmark is one set of tasks, so there is no suite; a non-null "
|
|
68
|
-
"value is refused. Omit it (an older SDK still sends it).")
|
|
69
|
-
|
|
70
|
-
|
|
71
64
|
|
|
72
65
|
|
|
73
66
|
trials: _Trials | dict[str, _Trials] | None = None
|
|
@@ -79,7 +72,12 @@ class EvalRequest(BaseModel):
|
|
|
79
72
|
policy_instances: int | None = Field(default=None, ge=1, le=64)
|
|
80
73
|
policy_gpu: str | None = Field(default=None, max_length=32)
|
|
81
74
|
benchmark_instances: int | None = Field(default=None, ge=1, le=64)
|
|
82
|
-
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
benchmark_gpu: str | None = Field(default=None, max_length=32, json_schema_extra={"deprecated": True},
|
|
79
|
+
description="Not accepted: an eval runs on the card its benchmark was registered "
|
|
80
|
+
"on, so a card is refused. Omit it (an older SDK still sends it).")
|
|
83
81
|
envs: int | None = Field(default=None, ge=1)
|
|
84
82
|
|
|
85
83
|
gpus: int = Field(default=1, ge=1, le=64, json_schema_extra={"deprecated": True},
|
|
@@ -284,18 +282,6 @@ class EvalStatus(str, Enum):
|
|
|
284
282
|
CANCELED = "canceled"
|
|
285
283
|
|
|
286
284
|
|
|
287
|
-
class SuiteAgg(BaseModel):
|
|
288
|
-
"""Per-suite aggregate of ONE user-named metric, as an eval from before contract 0.5.8 reported it: kept so those
|
|
289
|
-
reports stay readable. A benchmark is one set of tasks now, and no eval since reports one.
|
|
290
|
-
"""
|
|
291
|
-
|
|
292
|
-
model_config = ConfigDict(extra="forbid")
|
|
293
|
-
|
|
294
|
-
suite: str
|
|
295
|
-
value: float
|
|
296
|
-
n: int
|
|
297
|
-
|
|
298
|
-
|
|
299
285
|
class TaskAgg(BaseModel):
|
|
300
286
|
"""Per-task aggregate of ONE user-named metric — structured detail, not a bare dict."""
|
|
301
287
|
|
|
@@ -307,14 +293,13 @@ class TaskAgg(BaseModel):
|
|
|
307
293
|
|
|
308
294
|
|
|
309
295
|
class MetricAgg(BaseModel):
|
|
310
|
-
"""The aggregate of one user-named metric key across the eval — overall plus per-task breakdowns
|
|
311
|
-
|
|
296
|
+
"""The aggregate of one user-named metric key across the eval — overall plus per-task breakdowns and the sample
|
|
297
|
+
count. The platform aggregates whatever the user returned in `Step.metrics`;
|
|
312
298
|
it neither defines nor injects `success_rate` (success is just one key the user may or may not name)."""
|
|
313
299
|
|
|
314
300
|
model_config = ConfigDict(extra="forbid")
|
|
315
301
|
|
|
316
302
|
overall: float
|
|
317
|
-
per_suite: list[SuiteAgg] = Field(default_factory=list)
|
|
318
303
|
per_task: list[TaskAgg] = Field(default_factory=list)
|
|
319
304
|
n: int
|
|
320
305
|
|
|
@@ -341,8 +326,6 @@ class ArmReport(BaseModel):
|
|
|
341
326
|
|
|
342
327
|
params: dict[str, int | float | bool | str] = Field(default_factory=dict)
|
|
343
328
|
metrics: dict[str, MetricAgg] = Field(default_factory=dict)
|
|
344
|
-
|
|
345
|
-
intervals: dict[str, list[float]] = Field(default_factory=dict)
|
|
346
329
|
episodes: int = 0
|
|
347
330
|
failed: int = 0
|
|
348
331
|
not_run: int = 0
|
|
@@ -378,19 +361,6 @@ class ArmParams(BaseModel):
|
|
|
378
361
|
steps: int | None = None
|
|
379
362
|
|
|
380
363
|
|
|
381
|
-
class PairedDiff(BaseModel):
|
|
382
|
-
"""`compare`: one metric's difference between two arms over the seeds both ran (b - a), with its interval."""
|
|
383
|
-
|
|
384
|
-
model_config = ConfigDict(extra="forbid")
|
|
385
|
-
|
|
386
|
-
metric: str
|
|
387
|
-
arm_a: int
|
|
388
|
-
arm_b: int
|
|
389
|
-
n: int
|
|
390
|
-
mean_diff: float
|
|
391
|
-
interval: list[float]
|
|
392
|
-
|
|
393
|
-
|
|
394
364
|
class EvalTiming(BaseModel):
|
|
395
365
|
"""Where an eval's time went: cold starts, and per step the simulation, rendering, the policy's batched forward and
|
|
396
366
|
the network around it, plus the forward's average batch size."""
|
|
@@ -425,8 +395,8 @@ class EvalReport(BaseModel):
|
|
|
425
395
|
default_factory=list, description="Why episodes or the run failed, so an incomplete report says why"
|
|
426
396
|
)
|
|
427
397
|
|
|
398
|
+
|
|
428
399
|
arms: list[ArmReport] | None = None
|
|
429
|
-
paired: list[PairedDiff] | None = None
|
|
430
400
|
timing: EvalTiming | None = None
|
|
431
401
|
|
|
432
402
|
|
|
@@ -77,7 +77,6 @@ from sequence_base.contract.eval import (
|
|
|
77
77
|
LossyTransform,
|
|
78
78
|
MetricAgg,
|
|
79
79
|
RolloutResult,
|
|
80
|
-
SuiteAgg,
|
|
81
80
|
TaskAgg,
|
|
82
81
|
)
|
|
83
82
|
from sequence_base.contract.flags import seqflags_json_schema
|
|
@@ -167,7 +166,6 @@ WIRE_MODELS: dict[str, type[BaseModel]] = {
|
|
|
167
166
|
"EvalResponse": EvalResponse,
|
|
168
167
|
"EvalDryRunResponse": EvalDryRunResponse,
|
|
169
168
|
"RolloutResult": RolloutResult,
|
|
170
|
-
"SuiteAgg": SuiteAgg,
|
|
171
169
|
"TaskAgg": TaskAgg,
|
|
172
170
|
"MetricAgg": MetricAgg,
|
|
173
171
|
"LossyTransform": LossyTransform,
|
{sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/BenchmarkConditionSpec.json
RENAMED
|
@@ -19,18 +19,6 @@
|
|
|
19
19
|
"title": "Seed",
|
|
20
20
|
"type": "integer"
|
|
21
21
|
},
|
|
22
|
-
"suite": {
|
|
23
|
-
"anyOf": [
|
|
24
|
-
{
|
|
25
|
-
"type": "string"
|
|
26
|
-
},
|
|
27
|
-
{
|
|
28
|
-
"type": "null"
|
|
29
|
-
}
|
|
30
|
-
],
|
|
31
|
-
"default": null,
|
|
32
|
-
"title": "Suite"
|
|
33
|
-
},
|
|
34
22
|
"task": {
|
|
35
23
|
"title": "Task",
|
|
36
24
|
"type": "string"
|
|
@@ -62,5 +50,5 @@
|
|
|
62
50
|
],
|
|
63
51
|
"title": "BenchmarkConditionSpec",
|
|
64
52
|
"type": "object",
|
|
65
|
-
"x-contract-version": "0.5.
|
|
53
|
+
"x-contract-version": "0.5.11"
|
|
66
54
|
}
|
{sequence_base-0.4.10 → sequence_base-0.4.12}/sequence_base/schemas/BenchmarkRegisterRequest.json
RENAMED
|
@@ -164,18 +164,6 @@
|
|
|
164
164
|
"title": "Seed",
|
|
165
165
|
"type": "integer"
|
|
166
166
|
},
|
|
167
|
-
"suite": {
|
|
168
|
-
"anyOf": [
|
|
169
|
-
{
|
|
170
|
-
"type": "string"
|
|
171
|
-
},
|
|
172
|
-
{
|
|
173
|
-
"type": "null"
|
|
174
|
-
}
|
|
175
|
-
],
|
|
176
|
-
"default": null,
|
|
177
|
-
"title": "Suite"
|
|
178
|
-
},
|
|
179
167
|
"task": {
|
|
180
168
|
"title": "Task",
|
|
181
169
|
"type": "string"
|
|
@@ -954,13 +942,6 @@
|
|
|
954
942
|
"default": null,
|
|
955
943
|
"title": "Seeds Repeat"
|
|
956
944
|
},
|
|
957
|
-
"suite_of": {
|
|
958
|
-
"additionalProperties": {
|
|
959
|
-
"type": "string"
|
|
960
|
-
},
|
|
961
|
-
"title": "Suite Of",
|
|
962
|
-
"type": "object"
|
|
963
|
-
},
|
|
964
945
|
"task_descriptions": {
|
|
965
946
|
"additionalProperties": {
|
|
966
947
|
"type": "string"
|
|
@@ -991,5 +972,5 @@
|
|
|
991
972
|
],
|
|
992
973
|
"title": "BenchmarkRegisterRequest",
|
|
993
974
|
"type": "object",
|
|
994
|
-
"x-contract-version": "0.5.
|
|
975
|
+
"x-contract-version": "0.5.11"
|
|
995
976
|
}
|
|
@@ -368,7 +368,7 @@
|
|
|
368
368
|
],
|
|
369
369
|
"title": "SeqFlags",
|
|
370
370
|
"type": "integer",
|
|
371
|
-
"x-contract-version": "0.5.
|
|
371
|
+
"x-contract-version": "0.5.11",
|
|
372
372
|
"x-flag-names": {
|
|
373
373
|
"CHECK": 65536,
|
|
374
374
|
"INFER": 16,
|
|
@@ -625,5 +625,5 @@
|
|
|
625
625
|
],
|
|
626
626
|
"title": "BenchmarkSpec",
|
|
627
627
|
"type": "object",
|
|
628
|
-
"x-contract-version": "0.5.
|
|
628
|
+
"x-contract-version": "0.5.11"
|
|
629
629
|
}
|
|
@@ -997,6 +997,8 @@
|
|
|
997
997
|
}
|
|
998
998
|
],
|
|
999
999
|
"default": null,
|
|
1000
|
+
"deprecated": true,
|
|
1001
|
+
"description": "Not accepted: a deployment's workers run wherever there are cards, so a region is refused. Omit it (an older SDK still sends it).",
|
|
1000
1002
|
"title": "Region"
|
|
1001
1003
|
}
|
|
1002
1004
|
},
|
|
@@ -1224,5 +1226,5 @@
|
|
|
1224
1226
|
],
|
|
1225
1227
|
"title": "DeploymentRequest",
|
|
1226
1228
|
"type": "object",
|
|
1227
|
-
"x-contract-version": "0.5.
|
|
1229
|
+
"x-contract-version": "0.5.11"
|
|
1228
1230
|
}
|