sequence-base 0.3.4__tar.gz → 0.3.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sequence_base-0.3.4 → sequence_base-0.3.6}/PKG-INFO +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/pyproject.toml +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/__init__.py +3 -2
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/constants/version.py +3 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/contract/__init__.py +2 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/contract/benchmarks.py +27 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/contract/eval.py +6 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schema_export.py +2 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/ActionChunk.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/ActionContract.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/ActionStep.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/AdapterTransform.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/Autoscaler.json +1 -1
- sequence_base-0.3.6/sequence_base/schemas/BenchmarkConditionSpec.json +66 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/BenchmarkRegisterRequest.json +134 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/BenchmarkSpec.json +2 -2
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/BenchmarkStatusResponse.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/BootstrapResponse.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/Camera.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/ClampInfo.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/CodeBundle.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/Concurrency.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/Conditions.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/ConnectRequest.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/ConnectResponse.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/DeploymentAccepted.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/DeploymentPolicy.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/DeploymentRequest.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/DeploymentStatusResponse.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/Embodiment.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/ErrorBody.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/Estimate.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/EvalDryRunResponse.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/EvalReport.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/EvalRequest.json +42 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/EvalResponse.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/ImageFrame.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/ImageSpec.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/ImageStepApt.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/ImageStepEnv.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/ImageStepRun.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/ImageStepUvPip.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/Lease.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/LossyTransform.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/MetricAgg.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/Observation.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/ObservationContract.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/PolicyRegistration.json +2 -2
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/PolicySpec.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/Proprioception.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/RenderSpec.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/Resources.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/RolloutResult.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/SecretCreateRequest.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/SecretInfo.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/SecretRef.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/SeqFlags.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/Step.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/SuiteAgg.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/TaskAgg.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/Timeouts.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/UploadTarget.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/UsageReport.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/Variant.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/VolumeAttach.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/VolumeFile.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/VolumeInfo.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/VolumeMount.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/VolumeRef.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/VolumeUploadRequest.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/VolumeUploadResponse.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/WarmingError.json +1 -1
- {sequence_base-0.3.4 → sequence_base-0.3.6}/tests/test_contracts.py +3 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/.github/workflows/ci.yml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/.gitignore +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/README.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/README.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/REVIEW.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/START.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/accounts.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/billing.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/build/BUILD_LOG.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/build/goal_sha256.json +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/build/gs-build-wave.js +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/build/gs-spec-goals.js +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/container.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/contracts.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/deploy/PLAN.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/deploy/PROD_REORG.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/deploy/SCHEMA_COMPAT.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/deploy/archive/sequences-gw.genseq-506918.20261001.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/eval.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/gateway.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/A1.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/A2.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/A3.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/C1.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/C2.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/C3.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/C4.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/C5.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/G1.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/G2.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/G3.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/G4.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/G5.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/O1.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/O2.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/O3.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/O4.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/O5.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/R1.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/R2.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/R3.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/S1.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/S2.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/S3.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/S4.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/S5.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/S6.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/V1-storage.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/V2-user-quotas.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/W0.1.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/goals/W0.2.yaml +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/modal/README.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/modal/architecture.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/modal/connect.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/modal/cost.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/modal/custom-domain.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/modal/debugging.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/modal/goals.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/modal/limits.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/modal/plan.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/modal/white-label.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/modal/wiring.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/observability.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/references.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/docs/platform/sdk.md +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/auth.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/codec/__init__.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/codec/image.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/codec/resize.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/constants/__init__.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/constants/env.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/constants/headers.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/contract/accounts.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/contract/act.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/contract/connect.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/contract/deployments.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/contract/errors.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/contract/flags.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/contract/io.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/contract/policy.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/contract/volumes.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/errors/__init__.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/errors/transport.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/obs/__init__.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/obs/log.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/obs/rid.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/telemetry/__init__.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/telemetry/events.py +0 -0
- {sequence_base-0.3.4 → sequence_base-0.3.6}/tests/test_codec_roundtrip.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sequence-base
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.6
|
|
4
4
|
Summary: Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness.
|
|
5
5
|
Requires-Python: >=3.11
|
|
6
6
|
Requires-Dist: pydantic>=2.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sequence-base"
|
|
3
|
-
version = "0.3.
|
|
3
|
+
version = "0.3.6"
|
|
4
4
|
description = "Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness."
|
|
5
5
|
requires-python = ">=3.11"
|
|
6
6
|
dependencies = ["pydantic>=2.0"]
|
|
@@ -31,6 +31,7 @@ from sequence_base.contract import ( # noqa: F401 (re-export surface)
|
|
|
31
31
|
AdapterKind,
|
|
32
32
|
AdapterTransform,
|
|
33
33
|
Autoscaler,
|
|
34
|
+
BenchmarkConditionSpec,
|
|
34
35
|
BenchmarkRegisterRequest,
|
|
35
36
|
BenchmarkStatusResponse,
|
|
36
37
|
BenchmarkSpec,
|
|
@@ -126,7 +127,7 @@ __all__ = [
|
|
|
126
127
|
# contract 3 — connect / lease
|
|
127
128
|
"Lease", "ConnectRequest", "ConnectResponse",
|
|
128
129
|
# contract 5 — benchmark + eval
|
|
129
|
-
"Variant", "Conditions", "RenderSpec", "Step", "BenchmarkSpec", "BenchmarkRegisterRequest", "BenchmarkStatusResponse",
|
|
130
|
+
"Variant", "Conditions", "RenderSpec", "Step", "BenchmarkSpec", "BenchmarkConditionSpec", "BenchmarkRegisterRequest", "BenchmarkStatusResponse",
|
|
130
131
|
"EvalRequest", "ClampInfo", "Estimate", "EvalResponse", "EvalDryRunResponse", "RolloutResult",
|
|
131
132
|
"EvalStatus", "SuiteAgg", "TaskAgg", "MetricAgg", "LossyTransform", "EvalReport",
|
|
132
133
|
"AdapterKind", "AdapterTransform",
|
|
@@ -144,4 +145,4 @@ __all__ = [
|
|
|
144
145
|
"auth", "WorkerLogEvent",
|
|
145
146
|
]
|
|
146
147
|
|
|
147
|
-
__version__ = "0.3.
|
|
148
|
+
__version__ = "0.3.6"
|
|
@@ -6,6 +6,8 @@ PROTOCOL_VERSION = "1"
|
|
|
6
6
|
# across deployments / repos can tell which version of a shape it is validating against. MUST change
|
|
7
7
|
# whenever any shared wire shape changes (a field added / removed / retyped) — that is what makes
|
|
8
8
|
# contract evolution mechanically detectable instead of eyeballed. Bump on any shape change.
|
|
9
|
-
CONTRACT_VERSION = "0.4.
|
|
9
|
+
CONTRACT_VERSION = "0.4.5" # 0.4.4 -> 0.4.5: BenchmarkRegisterRequest.condition_list / timeout_s / default_adapter_ref (+ BenchmarkConditionSpec); EvalRequest.tasks / max_usd / rerun_of
|
|
10
|
+
# previous: 0.4.4 # 0.4.3 -> 0.4.4: BenchmarkRegisterRequest.secrets (by name) + score (has @seq.score)
|
|
11
|
+
# previous: 0.4.3 # 0.4.2 -> 0.4.3: I/O contract — ObservationContract/Camera/Embodiment on PolicySpec + DeploymentPolicy, Observation.state, ConnectRequest.embodiment, ConnectResponse.action/observation/embodiment
|
|
10
12
|
# previous: 0.4.2 # 0.4.1 -> 0.4.2: 契约7 volumes (uploads: VolumeFile/VolumeUploadRequest/UploadTarget/VolumeUploadResponse/VolumeInfo), weights accept vol://<name>[@v<n>]; DeploymentRequest name is a slug and weights need an origin
|
|
11
13
|
# previous: 0.4.1 # 0.4.0 -> 0.4.1: 契约2 deploy body completed during S3/G4 — DeploymentPolicy.secrets (name-only passthrough), DeploymentRequest.volumes: list[VolumeAttach{uri,mount,create_if_missing}], concurrency default materialized; telemetry events add load_start stage + emitted_at/rid/dedup_key (O4)
|
|
@@ -52,6 +52,7 @@ from sequence_base.contract.connect import ConnectRequest, ConnectResponse, Leas
|
|
|
52
52
|
|
|
53
53
|
# contract 5 — benchmark registration + eval run / report / adapter
|
|
54
54
|
from sequence_base.contract.benchmarks import (
|
|
55
|
+
BenchmarkConditionSpec,
|
|
55
56
|
BenchmarkRegisterRequest,
|
|
56
57
|
BenchmarkStatusResponse,
|
|
57
58
|
BenchmarkSpec,
|
|
@@ -117,7 +118,7 @@ __all__ = [
|
|
|
117
118
|
# contract 3
|
|
118
119
|
"Lease", "ConnectRequest", "ConnectResponse",
|
|
119
120
|
# contract 5
|
|
120
|
-
"Variant", "Conditions", "RenderSpec", "Step", "BenchmarkSpec", "BenchmarkRegisterRequest", "BenchmarkStatusResponse",
|
|
121
|
+
"Variant", "Conditions", "RenderSpec", "Step", "BenchmarkSpec", "BenchmarkConditionSpec", "BenchmarkRegisterRequest", "BenchmarkStatusResponse",
|
|
121
122
|
"EvalRequest", "ClampInfo", "Estimate", "EvalResponse", "EvalDryRunResponse", "RolloutResult",
|
|
122
123
|
"EvalStatus", "SuiteAgg", "TaskAgg", "MetricAgg", "LossyTransform", "EvalReport",
|
|
123
124
|
"AdapterKind", "AdapterTransform",
|
|
@@ -114,6 +114,20 @@ class BenchmarkSpec(BaseModel):
|
|
|
114
114
|
return self
|
|
115
115
|
|
|
116
116
|
|
|
117
|
+
class BenchmarkConditionSpec(BaseModel):
|
|
118
|
+
"""One episode setting exactly as the author declared it — a sparse list or a cross product, with its own step
|
|
119
|
+
cap, wall-clock limit and suite. {gs-plans/customization/benchmark.md "6. 其它在 SDK 里收下、在生产里丢掉的声明"}"""
|
|
120
|
+
|
|
121
|
+
model_config = ConfigDict(extra="forbid")
|
|
122
|
+
|
|
123
|
+
task: str
|
|
124
|
+
seed: int
|
|
125
|
+
variant: dict[str, str] = Field(default_factory=dict)
|
|
126
|
+
max_steps: int | None = Field(default=None, gt=0)
|
|
127
|
+
timeout_s: float | None = Field(default=None, gt=0)
|
|
128
|
+
suite: str | None = None
|
|
129
|
+
|
|
130
|
+
|
|
117
131
|
class BenchmarkRegisterRequest(BaseModel):
|
|
118
132
|
"""POST /v1/benchmarks — register body, same types as the spec (ImageSpec / Conditions / VolumeRef /
|
|
119
133
|
RenderSpec). Carries assets / render / cpu / memory_gb / max_steps so the gateway can build the image,
|
|
@@ -139,6 +153,19 @@ class BenchmarkRegisterRequest(BaseModel):
|
|
|
139
153
|
# The action contract the benchmark's sim consumes; eval compares it with the policy's and requires an
|
|
140
154
|
# adapter when they differ.
|
|
141
155
|
action_contract: ActionContract | None = None
|
|
156
|
+
# Credentials the sim's own code needs (a dataset token), by name only — injected into the build check and
|
|
157
|
+
# every eval shard, never into the policy. {gs-plans/customization/benchmark.md "A.1 SECRETS"}
|
|
158
|
+
secrets: list[SecretRef] = Field(default_factory=list)
|
|
159
|
+
# Whether the benchmark defines @seq.score — the platform then calls it per group (overall, each suite, each
|
|
160
|
+
# task) after every episode has run, and its keys override those groups' means. {benchmark.md "A.2"}
|
|
161
|
+
score: bool = False
|
|
162
|
+
# Every condition as declared (a sparse list stays sparse; each keeps its own max_steps / timeout_s / suite),
|
|
163
|
+
# where `conditions` can only say tasks x seeds x variants. The eval runs these when present.
|
|
164
|
+
condition_list: list[BenchmarkConditionSpec] | None = Field(default=None, max_length=5000)
|
|
165
|
+
# An episode's wall-clock limit (seconds) when a condition sets none; past it the episode ends truncated.
|
|
166
|
+
timeout_s: float | None = Field(default=None, gt=0)
|
|
167
|
+
# The adapter (module:Class in the benchmark's own code) an eval uses when it names none.
|
|
168
|
+
default_adapter_ref: str | None = None
|
|
142
169
|
|
|
143
170
|
|
|
144
171
|
class BenchmarkStatusResponse(BaseModel):
|
|
@@ -47,6 +47,12 @@ class EvalRequest(BaseModel):
|
|
|
47
47
|
adapter: str | None = None
|
|
48
48
|
compare: str | None = Field(default=None, description="Another model id to compare against")
|
|
49
49
|
dry_run: bool = False
|
|
50
|
+
# Only these tasks (of the selected suite / trials). {gs-plans/customization/benchmark.md "子集"}
|
|
51
|
+
tasks: list[str] | None = Field(default=None, max_length=1000)
|
|
52
|
+
# The most this eval may spend on GPU time: it stops there and reports what ran (incomplete).
|
|
53
|
+
max_usd: float | None = Field(default=None, gt=0)
|
|
54
|
+
# Run again only the episodes of that earlier eval that errored; the report covers both runs.
|
|
55
|
+
rerun_of: str | None = None
|
|
50
56
|
|
|
51
57
|
|
|
52
58
|
class ClampInfo(BaseModel):
|
|
@@ -35,6 +35,7 @@ from sequence_base.contract.act import (
|
|
|
35
35
|
Proprioception,
|
|
36
36
|
)
|
|
37
37
|
from sequence_base.contract.benchmarks import (
|
|
38
|
+
BenchmarkConditionSpec,
|
|
38
39
|
BenchmarkRegisterRequest,
|
|
39
40
|
BenchmarkStatusResponse,
|
|
40
41
|
BenchmarkSpec,
|
|
@@ -141,6 +142,7 @@ WIRE_MODELS: dict[str, type[BaseModel]] = {
|
|
|
141
142
|
"RenderSpec": RenderSpec,
|
|
142
143
|
"Step": Step,
|
|
143
144
|
"BenchmarkSpec": BenchmarkSpec,
|
|
145
|
+
"BenchmarkConditionSpec": BenchmarkConditionSpec,
|
|
144
146
|
"BenchmarkRegisterRequest": BenchmarkRegisterRequest,
|
|
145
147
|
"BenchmarkStatusResponse": BenchmarkStatusResponse,
|
|
146
148
|
# contract 5 — eval run / report / adapter
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
{
|
|
2
|
+
"additionalProperties": false,
|
|
3
|
+
"description": "One episode setting exactly as the author declared it — a sparse list or a cross product, with its own step\ncap, wall-clock limit and suite. {gs-plans/customization/benchmark.md \"6. 其它在 SDK 里收下、在生产里丢掉的声明\"}",
|
|
4
|
+
"properties": {
|
|
5
|
+
"max_steps": {
|
|
6
|
+
"anyOf": [
|
|
7
|
+
{
|
|
8
|
+
"exclusiveMinimum": 0,
|
|
9
|
+
"type": "integer"
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
"type": "null"
|
|
13
|
+
}
|
|
14
|
+
],
|
|
15
|
+
"default": null,
|
|
16
|
+
"title": "Max Steps"
|
|
17
|
+
},
|
|
18
|
+
"seed": {
|
|
19
|
+
"title": "Seed",
|
|
20
|
+
"type": "integer"
|
|
21
|
+
},
|
|
22
|
+
"suite": {
|
|
23
|
+
"anyOf": [
|
|
24
|
+
{
|
|
25
|
+
"type": "string"
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
"type": "null"
|
|
29
|
+
}
|
|
30
|
+
],
|
|
31
|
+
"default": null,
|
|
32
|
+
"title": "Suite"
|
|
33
|
+
},
|
|
34
|
+
"task": {
|
|
35
|
+
"title": "Task",
|
|
36
|
+
"type": "string"
|
|
37
|
+
},
|
|
38
|
+
"timeout_s": {
|
|
39
|
+
"anyOf": [
|
|
40
|
+
{
|
|
41
|
+
"exclusiveMinimum": 0,
|
|
42
|
+
"type": "number"
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"type": "null"
|
|
46
|
+
}
|
|
47
|
+
],
|
|
48
|
+
"default": null,
|
|
49
|
+
"title": "Timeout S"
|
|
50
|
+
},
|
|
51
|
+
"variant": {
|
|
52
|
+
"additionalProperties": {
|
|
53
|
+
"type": "string"
|
|
54
|
+
},
|
|
55
|
+
"title": "Variant",
|
|
56
|
+
"type": "object"
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
"required": [
|
|
60
|
+
"task",
|
|
61
|
+
"seed"
|
|
62
|
+
],
|
|
63
|
+
"title": "BenchmarkConditionSpec",
|
|
64
|
+
"type": "object",
|
|
65
|
+
"x-contract-version": "0.4.5"
|
|
66
|
+
}
|
{sequence_base-0.3.4 → sequence_base-0.3.6}/sequence_base/schemas/BenchmarkRegisterRequest.json
RENAMED
|
@@ -40,6 +40,71 @@
|
|
|
40
40
|
"title": "ActionSpace",
|
|
41
41
|
"type": "string"
|
|
42
42
|
},
|
|
43
|
+
"BenchmarkConditionSpec": {
|
|
44
|
+
"additionalProperties": false,
|
|
45
|
+
"description": "One episode setting exactly as the author declared it — a sparse list or a cross product, with its own step\ncap, wall-clock limit and suite. {gs-plans/customization/benchmark.md \"6. 其它在 SDK 里收下、在生产里丢掉的声明\"}",
|
|
46
|
+
"properties": {
|
|
47
|
+
"max_steps": {
|
|
48
|
+
"anyOf": [
|
|
49
|
+
{
|
|
50
|
+
"exclusiveMinimum": 0,
|
|
51
|
+
"type": "integer"
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
"type": "null"
|
|
55
|
+
}
|
|
56
|
+
],
|
|
57
|
+
"default": null,
|
|
58
|
+
"title": "Max Steps"
|
|
59
|
+
},
|
|
60
|
+
"seed": {
|
|
61
|
+
"title": "Seed",
|
|
62
|
+
"type": "integer"
|
|
63
|
+
},
|
|
64
|
+
"suite": {
|
|
65
|
+
"anyOf": [
|
|
66
|
+
{
|
|
67
|
+
"type": "string"
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"type": "null"
|
|
71
|
+
}
|
|
72
|
+
],
|
|
73
|
+
"default": null,
|
|
74
|
+
"title": "Suite"
|
|
75
|
+
},
|
|
76
|
+
"task": {
|
|
77
|
+
"title": "Task",
|
|
78
|
+
"type": "string"
|
|
79
|
+
},
|
|
80
|
+
"timeout_s": {
|
|
81
|
+
"anyOf": [
|
|
82
|
+
{
|
|
83
|
+
"exclusiveMinimum": 0,
|
|
84
|
+
"type": "number"
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
"type": "null"
|
|
88
|
+
}
|
|
89
|
+
],
|
|
90
|
+
"default": null,
|
|
91
|
+
"title": "Timeout S"
|
|
92
|
+
},
|
|
93
|
+
"variant": {
|
|
94
|
+
"additionalProperties": {
|
|
95
|
+
"type": "string"
|
|
96
|
+
},
|
|
97
|
+
"title": "Variant",
|
|
98
|
+
"type": "object"
|
|
99
|
+
}
|
|
100
|
+
},
|
|
101
|
+
"required": [
|
|
102
|
+
"task",
|
|
103
|
+
"seed"
|
|
104
|
+
],
|
|
105
|
+
"title": "BenchmarkConditionSpec",
|
|
106
|
+
"type": "object"
|
|
107
|
+
},
|
|
43
108
|
"CodeBundle": {
|
|
44
109
|
"additionalProperties": false,
|
|
45
110
|
"description": "The policy's own source, shipped with the deploy so the worker can import `policy.handler_ref`\n(what Modal's automount does for the entrypoint module). A gzip'd tar of the handler's top-level module\nor package, base64 on the wire, with the sha256 of the decoded bytes so a truncated or altered upload\nis refused at the edge instead of failing as an ImportError on a GPU.",
|
|
@@ -240,6 +305,21 @@
|
|
|
240
305
|
"title": "RenderSpec",
|
|
241
306
|
"type": "object"
|
|
242
307
|
},
|
|
308
|
+
"SecretRef": {
|
|
309
|
+
"additionalProperties": false,
|
|
310
|
+
"description": "A by-name reference to a secret (PolicySpec.secrets / BenchmarkSpec.secrets). Name only — the\nvalue lives in Secret Manager and is injected at worker bootstrap. Never carries a value.",
|
|
311
|
+
"properties": {
|
|
312
|
+
"name": {
|
|
313
|
+
"title": "Name",
|
|
314
|
+
"type": "string"
|
|
315
|
+
}
|
|
316
|
+
},
|
|
317
|
+
"required": [
|
|
318
|
+
"name"
|
|
319
|
+
],
|
|
320
|
+
"title": "SecretRef",
|
|
321
|
+
"type": "object"
|
|
322
|
+
},
|
|
243
323
|
"Variant": {
|
|
244
324
|
"additionalProperties": false,
|
|
245
325
|
"description": "One structured variant axis of the eval matrix (e.g. axis=\"lighting\", values=[\"day\",\"night\"]).",
|
|
@@ -323,6 +403,22 @@
|
|
|
323
403
|
],
|
|
324
404
|
"default": null
|
|
325
405
|
},
|
|
406
|
+
"condition_list": {
|
|
407
|
+
"anyOf": [
|
|
408
|
+
{
|
|
409
|
+
"items": {
|
|
410
|
+
"$ref": "#/$defs/BenchmarkConditionSpec"
|
|
411
|
+
},
|
|
412
|
+
"maxItems": 5000,
|
|
413
|
+
"type": "array"
|
|
414
|
+
},
|
|
415
|
+
{
|
|
416
|
+
"type": "null"
|
|
417
|
+
}
|
|
418
|
+
],
|
|
419
|
+
"default": null,
|
|
420
|
+
"title": "Condition List"
|
|
421
|
+
},
|
|
326
422
|
"conditions": {
|
|
327
423
|
"$ref": "#/$defs/Conditions"
|
|
328
424
|
},
|
|
@@ -338,6 +434,18 @@
|
|
|
338
434
|
"default": null,
|
|
339
435
|
"title": "Cpu"
|
|
340
436
|
},
|
|
437
|
+
"default_adapter_ref": {
|
|
438
|
+
"anyOf": [
|
|
439
|
+
{
|
|
440
|
+
"type": "string"
|
|
441
|
+
},
|
|
442
|
+
{
|
|
443
|
+
"type": "null"
|
|
444
|
+
}
|
|
445
|
+
],
|
|
446
|
+
"default": null,
|
|
447
|
+
"title": "Default Adapter Ref"
|
|
448
|
+
},
|
|
341
449
|
"handler_ref": {
|
|
342
450
|
"title": "Handler Ref",
|
|
343
451
|
"type": "string"
|
|
@@ -376,12 +484,37 @@
|
|
|
376
484
|
],
|
|
377
485
|
"default": null
|
|
378
486
|
},
|
|
487
|
+
"score": {
|
|
488
|
+
"default": false,
|
|
489
|
+
"title": "Score",
|
|
490
|
+
"type": "boolean"
|
|
491
|
+
},
|
|
492
|
+
"secrets": {
|
|
493
|
+
"items": {
|
|
494
|
+
"$ref": "#/$defs/SecretRef"
|
|
495
|
+
},
|
|
496
|
+
"title": "Secrets",
|
|
497
|
+
"type": "array"
|
|
498
|
+
},
|
|
379
499
|
"suite_of": {
|
|
380
500
|
"additionalProperties": {
|
|
381
501
|
"type": "string"
|
|
382
502
|
},
|
|
383
503
|
"title": "Suite Of",
|
|
384
504
|
"type": "object"
|
|
505
|
+
},
|
|
506
|
+
"timeout_s": {
|
|
507
|
+
"anyOf": [
|
|
508
|
+
{
|
|
509
|
+
"exclusiveMinimum": 0,
|
|
510
|
+
"type": "number"
|
|
511
|
+
},
|
|
512
|
+
{
|
|
513
|
+
"type": "null"
|
|
514
|
+
}
|
|
515
|
+
],
|
|
516
|
+
"default": null,
|
|
517
|
+
"title": "Timeout S"
|
|
385
518
|
}
|
|
386
519
|
},
|
|
387
520
|
"required": [
|
|
@@ -393,5 +526,5 @@
|
|
|
393
526
|
],
|
|
394
527
|
"title": "BenchmarkRegisterRequest",
|
|
395
528
|
"type": "object",
|
|
396
|
-
"x-contract-version": "0.4.
|
|
529
|
+
"x-contract-version": "0.4.5"
|
|
397
530
|
}
|
|
@@ -207,7 +207,7 @@
|
|
|
207
207
|
],
|
|
208
208
|
"title": "SeqFlags",
|
|
209
209
|
"type": "integer",
|
|
210
|
-
"x-contract-version": "0.4.
|
|
210
|
+
"x-contract-version": "0.4.5",
|
|
211
211
|
"x-flag-names": {
|
|
212
212
|
"CHECK": 65536,
|
|
213
213
|
"INFER": 16,
|
|
@@ -397,5 +397,5 @@
|
|
|
397
397
|
],
|
|
398
398
|
"title": "BenchmarkSpec",
|
|
399
399
|
"type": "object",
|
|
400
|
-
"x-contract-version": "0.4.
|
|
400
|
+
"x-contract-version": "0.4.5"
|
|
401
401
|
}
|
|
@@ -60,10 +60,35 @@
|
|
|
60
60
|
"default": null,
|
|
61
61
|
"title": "Gpus"
|
|
62
62
|
},
|
|
63
|
+
"max_usd": {
|
|
64
|
+
"anyOf": [
|
|
65
|
+
{
|
|
66
|
+
"exclusiveMinimum": 0,
|
|
67
|
+
"type": "number"
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"type": "null"
|
|
71
|
+
}
|
|
72
|
+
],
|
|
73
|
+
"default": null,
|
|
74
|
+
"title": "Max Usd"
|
|
75
|
+
},
|
|
63
76
|
"model": {
|
|
64
77
|
"title": "Model",
|
|
65
78
|
"type": "string"
|
|
66
79
|
},
|
|
80
|
+
"rerun_of": {
|
|
81
|
+
"anyOf": [
|
|
82
|
+
{
|
|
83
|
+
"type": "string"
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"type": "null"
|
|
87
|
+
}
|
|
88
|
+
],
|
|
89
|
+
"default": null,
|
|
90
|
+
"title": "Rerun Of"
|
|
91
|
+
},
|
|
67
92
|
"suite": {
|
|
68
93
|
"anyOf": [
|
|
69
94
|
{
|
|
@@ -76,6 +101,22 @@
|
|
|
76
101
|
"default": null,
|
|
77
102
|
"title": "Suite"
|
|
78
103
|
},
|
|
104
|
+
"tasks": {
|
|
105
|
+
"anyOf": [
|
|
106
|
+
{
|
|
107
|
+
"items": {
|
|
108
|
+
"type": "string"
|
|
109
|
+
},
|
|
110
|
+
"maxItems": 1000,
|
|
111
|
+
"type": "array"
|
|
112
|
+
},
|
|
113
|
+
{
|
|
114
|
+
"type": "null"
|
|
115
|
+
}
|
|
116
|
+
],
|
|
117
|
+
"default": null,
|
|
118
|
+
"title": "Tasks"
|
|
119
|
+
},
|
|
79
120
|
"trials": {
|
|
80
121
|
"anyOf": [
|
|
81
122
|
{
|
|
@@ -107,5 +148,5 @@
|
|
|
107
148
|
],
|
|
108
149
|
"title": "EvalRequest",
|
|
109
150
|
"type": "object",
|
|
110
|
-
"x-contract-version": "0.4.
|
|
151
|
+
"x-contract-version": "0.4.5"
|
|
111
152
|
}
|