sequence-base 0.3.5__tar.gz → 0.3.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sequence_base-0.3.5 → sequence_base-0.3.6}/PKG-INFO +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/pyproject.toml +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/__init__.py +3 -2
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/constants/version.py +2 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/contract/__init__.py +2 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/contract/benchmarks.py +21 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/contract/eval.py +6 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schema_export.py +2 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/ActionChunk.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/ActionContract.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/ActionStep.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/AdapterTransform.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/Autoscaler.json +1 -1
- sequence_base-0.3.6/sequence_base/schemas/BenchmarkConditionSpec.json +66 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/BenchmarkRegisterRequest.json +107 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/BenchmarkSpec.json +2 -2
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/BenchmarkStatusResponse.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/BootstrapResponse.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/Camera.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/ClampInfo.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/CodeBundle.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/Concurrency.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/Conditions.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/ConnectRequest.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/ConnectResponse.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/DeploymentAccepted.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/DeploymentPolicy.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/DeploymentRequest.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/DeploymentStatusResponse.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/Embodiment.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/ErrorBody.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/Estimate.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/EvalDryRunResponse.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/EvalReport.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/EvalRequest.json +42 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/EvalResponse.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/ImageFrame.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/ImageSpec.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/ImageStepApt.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/ImageStepEnv.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/ImageStepRun.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/ImageStepUvPip.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/Lease.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/LossyTransform.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/MetricAgg.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/Observation.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/ObservationContract.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/PolicyRegistration.json +2 -2
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/PolicySpec.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/Proprioception.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/RenderSpec.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/Resources.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/RolloutResult.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/SecretCreateRequest.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/SecretInfo.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/SecretRef.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/SeqFlags.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/Step.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/SuiteAgg.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/TaskAgg.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/Timeouts.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/UploadTarget.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/UsageReport.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/Variant.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/VolumeAttach.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/VolumeFile.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/VolumeInfo.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/VolumeMount.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/VolumeRef.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/VolumeUploadRequest.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/VolumeUploadResponse.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/WarmingError.json +1 -1
- {sequence_base-0.3.5 → sequence_base-0.3.6}/tests/test_contracts.py +3 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/.github/workflows/ci.yml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/.gitignore +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/README.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/README.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/REVIEW.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/START.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/accounts.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/billing.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/build/BUILD_LOG.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/build/goal_sha256.json +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/build/gs-build-wave.js +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/build/gs-spec-goals.js +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/container.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/contracts.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/deploy/PLAN.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/deploy/PROD_REORG.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/deploy/SCHEMA_COMPAT.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/deploy/archive/sequences-gw.genseq-506918.20261001.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/eval.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/gateway.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/A1.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/A2.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/A3.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/C1.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/C2.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/C3.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/C4.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/C5.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/G1.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/G2.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/G3.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/G4.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/G5.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/O1.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/O2.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/O3.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/O4.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/O5.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/R1.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/R2.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/R3.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/S1.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/S2.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/S3.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/S4.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/S5.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/S6.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/V1-storage.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/V2-user-quotas.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/W0.1.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/goals/W0.2.yaml +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/modal/README.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/modal/architecture.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/modal/connect.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/modal/cost.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/modal/custom-domain.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/modal/debugging.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/modal/goals.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/modal/limits.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/modal/plan.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/modal/white-label.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/modal/wiring.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/observability.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/references.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/docs/platform/sdk.md +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/auth.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/codec/__init__.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/codec/image.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/codec/resize.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/constants/__init__.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/constants/env.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/constants/headers.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/contract/accounts.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/contract/act.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/contract/connect.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/contract/deployments.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/contract/errors.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/contract/flags.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/contract/io.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/contract/policy.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/contract/volumes.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/errors/__init__.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/errors/transport.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/obs/__init__.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/obs/log.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/obs/rid.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/telemetry/__init__.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/telemetry/events.py +0 -0
- {sequence_base-0.3.5 → sequence_base-0.3.6}/tests/test_codec_roundtrip.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sequence-base
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.6
|
|
4
4
|
Summary: Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness.
|
|
5
5
|
Requires-Python: >=3.11
|
|
6
6
|
Requires-Dist: pydantic>=2.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sequence-base"
|
|
3
|
-
version = "0.3.
|
|
3
|
+
version = "0.3.6"
|
|
4
4
|
description = "Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness."
|
|
5
5
|
requires-python = ">=3.11"
|
|
6
6
|
dependencies = ["pydantic>=2.0"]
|
|
@@ -31,6 +31,7 @@ from sequence_base.contract import ( # noqa: F401 (re-export surface)
|
|
|
31
31
|
AdapterKind,
|
|
32
32
|
AdapterTransform,
|
|
33
33
|
Autoscaler,
|
|
34
|
+
BenchmarkConditionSpec,
|
|
34
35
|
BenchmarkRegisterRequest,
|
|
35
36
|
BenchmarkStatusResponse,
|
|
36
37
|
BenchmarkSpec,
|
|
@@ -126,7 +127,7 @@ __all__ = [
|
|
|
126
127
|
# contract 3 — connect / lease
|
|
127
128
|
"Lease", "ConnectRequest", "ConnectResponse",
|
|
128
129
|
# contract 5 — benchmark + eval
|
|
129
|
-
"Variant", "Conditions", "RenderSpec", "Step", "BenchmarkSpec", "BenchmarkRegisterRequest", "BenchmarkStatusResponse",
|
|
130
|
+
"Variant", "Conditions", "RenderSpec", "Step", "BenchmarkSpec", "BenchmarkConditionSpec", "BenchmarkRegisterRequest", "BenchmarkStatusResponse",
|
|
130
131
|
"EvalRequest", "ClampInfo", "Estimate", "EvalResponse", "EvalDryRunResponse", "RolloutResult",
|
|
131
132
|
"EvalStatus", "SuiteAgg", "TaskAgg", "MetricAgg", "LossyTransform", "EvalReport",
|
|
132
133
|
"AdapterKind", "AdapterTransform",
|
|
@@ -144,4 +145,4 @@ __all__ = [
|
|
|
144
145
|
"auth", "WorkerLogEvent",
|
|
145
146
|
]
|
|
146
147
|
|
|
147
|
-
__version__ = "0.3.
|
|
148
|
+
__version__ = "0.3.6"
|
|
@@ -6,7 +6,8 @@ PROTOCOL_VERSION = "1"
|
|
|
6
6
|
# across deployments / repos can tell which version of a shape it is validating against. MUST change
|
|
7
7
|
# whenever any shared wire shape changes (a field added / removed / retyped) — that is what makes
|
|
8
8
|
# contract evolution mechanically detectable instead of eyeballed. Bump on any shape change.
|
|
9
|
-
CONTRACT_VERSION = "0.4.
|
|
9
|
+
CONTRACT_VERSION = "0.4.5" # 0.4.4 -> 0.4.5: BenchmarkRegisterRequest.condition_list / timeout_s / default_adapter_ref (+ BenchmarkConditionSpec); EvalRequest.tasks / max_usd / rerun_of
|
|
10
|
+
# previous: 0.4.4 # 0.4.3 -> 0.4.4: BenchmarkRegisterRequest.secrets (by name) + score (has @seq.score)
|
|
10
11
|
# previous: 0.4.3 # 0.4.2 -> 0.4.3: I/O contract — ObservationContract/Camera/Embodiment on PolicySpec + DeploymentPolicy, Observation.state, ConnectRequest.embodiment, ConnectResponse.action/observation/embodiment
|
|
11
12
|
# previous: 0.4.2 # 0.4.1 -> 0.4.2: 契约7 volumes (uploads: VolumeFile/VolumeUploadRequest/UploadTarget/VolumeUploadResponse/VolumeInfo), weights accept vol://<name>[@v<n>]; DeploymentRequest name is a slug and weights need an origin
|
|
12
13
|
# previous: 0.4.1 # 0.4.0 -> 0.4.1: 契约2 deploy body completed during S3/G4 — DeploymentPolicy.secrets (name-only passthrough), DeploymentRequest.volumes: list[VolumeAttach{uri,mount,create_if_missing}], concurrency default materialized; telemetry events add load_start stage + emitted_at/rid/dedup_key (O4)
|
|
@@ -52,6 +52,7 @@ from sequence_base.contract.connect import ConnectRequest, ConnectResponse, Leas
|
|
|
52
52
|
|
|
53
53
|
# contract 5 — benchmark registration + eval run / report / adapter
|
|
54
54
|
from sequence_base.contract.benchmarks import (
|
|
55
|
+
BenchmarkConditionSpec,
|
|
55
56
|
BenchmarkRegisterRequest,
|
|
56
57
|
BenchmarkStatusResponse,
|
|
57
58
|
BenchmarkSpec,
|
|
@@ -117,7 +118,7 @@ __all__ = [
|
|
|
117
118
|
# contract 3
|
|
118
119
|
"Lease", "ConnectRequest", "ConnectResponse",
|
|
119
120
|
# contract 5
|
|
120
|
-
"Variant", "Conditions", "RenderSpec", "Step", "BenchmarkSpec", "BenchmarkRegisterRequest", "BenchmarkStatusResponse",
|
|
121
|
+
"Variant", "Conditions", "RenderSpec", "Step", "BenchmarkSpec", "BenchmarkConditionSpec", "BenchmarkRegisterRequest", "BenchmarkStatusResponse",
|
|
121
122
|
"EvalRequest", "ClampInfo", "Estimate", "EvalResponse", "EvalDryRunResponse", "RolloutResult",
|
|
122
123
|
"EvalStatus", "SuiteAgg", "TaskAgg", "MetricAgg", "LossyTransform", "EvalReport",
|
|
123
124
|
"AdapterKind", "AdapterTransform",
|
|
@@ -114,6 +114,20 @@ class BenchmarkSpec(BaseModel):
|
|
|
114
114
|
return self
|
|
115
115
|
|
|
116
116
|
|
|
117
|
+
class BenchmarkConditionSpec(BaseModel):
|
|
118
|
+
"""One episode setting exactly as the author declared it — a sparse list or a cross product, with its own step
|
|
119
|
+
cap, wall-clock limit and suite. {gs-plans/customization/benchmark.md "6. 其它在 SDK 里收下、在生产里丢掉的声明"}"""
|
|
120
|
+
|
|
121
|
+
model_config = ConfigDict(extra="forbid")
|
|
122
|
+
|
|
123
|
+
task: str
|
|
124
|
+
seed: int
|
|
125
|
+
variant: dict[str, str] = Field(default_factory=dict)
|
|
126
|
+
max_steps: int | None = Field(default=None, gt=0)
|
|
127
|
+
timeout_s: float | None = Field(default=None, gt=0)
|
|
128
|
+
suite: str | None = None
|
|
129
|
+
|
|
130
|
+
|
|
117
131
|
class BenchmarkRegisterRequest(BaseModel):
|
|
118
132
|
"""POST /v1/benchmarks — register body, same types as the spec (ImageSpec / Conditions / VolumeRef /
|
|
119
133
|
RenderSpec). Carries assets / render / cpu / memory_gb / max_steps so the gateway can build the image,
|
|
@@ -145,6 +159,13 @@ class BenchmarkRegisterRequest(BaseModel):
|
|
|
145
159
|
# Whether the benchmark defines @seq.score — the platform then calls it per group (overall, each suite, each
|
|
146
160
|
# task) after every episode has run, and its keys override those groups' means. {benchmark.md "A.2"}
|
|
147
161
|
score: bool = False
|
|
162
|
+
# Every condition as declared (a sparse list stays sparse; each keeps its own max_steps / timeout_s / suite),
|
|
163
|
+
# where `conditions` can only say tasks x seeds x variants. The eval runs these when present.
|
|
164
|
+
condition_list: list[BenchmarkConditionSpec] | None = Field(default=None, max_length=5000)
|
|
165
|
+
# An episode's wall-clock limit (seconds) when a condition sets none; past it the episode ends truncated.
|
|
166
|
+
timeout_s: float | None = Field(default=None, gt=0)
|
|
167
|
+
# The adapter (module:Class in the benchmark's own code) an eval uses when it names none.
|
|
168
|
+
default_adapter_ref: str | None = None
|
|
148
169
|
|
|
149
170
|
|
|
150
171
|
class BenchmarkStatusResponse(BaseModel):
|
|
@@ -47,6 +47,12 @@ class EvalRequest(BaseModel):
|
|
|
47
47
|
adapter: str | None = None
|
|
48
48
|
compare: str | None = Field(default=None, description="Another model id to compare against")
|
|
49
49
|
dry_run: bool = False
|
|
50
|
+
# Only these tasks (of the selected suite / trials). {gs-plans/customization/benchmark.md "子集"}
|
|
51
|
+
tasks: list[str] | None = Field(default=None, max_length=1000)
|
|
52
|
+
# The most this eval may spend on GPU time: it stops there and reports what ran (incomplete).
|
|
53
|
+
max_usd: float | None = Field(default=None, gt=0)
|
|
54
|
+
# Run again only the episodes of that earlier eval that errored; the report covers both runs.
|
|
55
|
+
rerun_of: str | None = None
|
|
50
56
|
|
|
51
57
|
|
|
52
58
|
class ClampInfo(BaseModel):
|
|
@@ -35,6 +35,7 @@ from sequence_base.contract.act import (
|
|
|
35
35
|
Proprioception,
|
|
36
36
|
)
|
|
37
37
|
from sequence_base.contract.benchmarks import (
|
|
38
|
+
BenchmarkConditionSpec,
|
|
38
39
|
BenchmarkRegisterRequest,
|
|
39
40
|
BenchmarkStatusResponse,
|
|
40
41
|
BenchmarkSpec,
|
|
@@ -141,6 +142,7 @@ WIRE_MODELS: dict[str, type[BaseModel]] = {
|
|
|
141
142
|
"RenderSpec": RenderSpec,
|
|
142
143
|
"Step": Step,
|
|
143
144
|
"BenchmarkSpec": BenchmarkSpec,
|
|
145
|
+
"BenchmarkConditionSpec": BenchmarkConditionSpec,
|
|
144
146
|
"BenchmarkRegisterRequest": BenchmarkRegisterRequest,
|
|
145
147
|
"BenchmarkStatusResponse": BenchmarkStatusResponse,
|
|
146
148
|
# contract 5 — eval run / report / adapter
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
{
|
|
2
|
+
"additionalProperties": false,
|
|
3
|
+
"description": "One episode setting exactly as the author declared it — a sparse list or a cross product, with its own step\ncap, wall-clock limit and suite. {gs-plans/customization/benchmark.md \"6. 其它在 SDK 里收下、在生产里丢掉的声明\"}",
|
|
4
|
+
"properties": {
|
|
5
|
+
"max_steps": {
|
|
6
|
+
"anyOf": [
|
|
7
|
+
{
|
|
8
|
+
"exclusiveMinimum": 0,
|
|
9
|
+
"type": "integer"
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
"type": "null"
|
|
13
|
+
}
|
|
14
|
+
],
|
|
15
|
+
"default": null,
|
|
16
|
+
"title": "Max Steps"
|
|
17
|
+
},
|
|
18
|
+
"seed": {
|
|
19
|
+
"title": "Seed",
|
|
20
|
+
"type": "integer"
|
|
21
|
+
},
|
|
22
|
+
"suite": {
|
|
23
|
+
"anyOf": [
|
|
24
|
+
{
|
|
25
|
+
"type": "string"
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
"type": "null"
|
|
29
|
+
}
|
|
30
|
+
],
|
|
31
|
+
"default": null,
|
|
32
|
+
"title": "Suite"
|
|
33
|
+
},
|
|
34
|
+
"task": {
|
|
35
|
+
"title": "Task",
|
|
36
|
+
"type": "string"
|
|
37
|
+
},
|
|
38
|
+
"timeout_s": {
|
|
39
|
+
"anyOf": [
|
|
40
|
+
{
|
|
41
|
+
"exclusiveMinimum": 0,
|
|
42
|
+
"type": "number"
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"type": "null"
|
|
46
|
+
}
|
|
47
|
+
],
|
|
48
|
+
"default": null,
|
|
49
|
+
"title": "Timeout S"
|
|
50
|
+
},
|
|
51
|
+
"variant": {
|
|
52
|
+
"additionalProperties": {
|
|
53
|
+
"type": "string"
|
|
54
|
+
},
|
|
55
|
+
"title": "Variant",
|
|
56
|
+
"type": "object"
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
"required": [
|
|
60
|
+
"task",
|
|
61
|
+
"seed"
|
|
62
|
+
],
|
|
63
|
+
"title": "BenchmarkConditionSpec",
|
|
64
|
+
"type": "object",
|
|
65
|
+
"x-contract-version": "0.4.5"
|
|
66
|
+
}
|
{sequence_base-0.3.5 → sequence_base-0.3.6}/sequence_base/schemas/BenchmarkRegisterRequest.json
RENAMED
|
@@ -40,6 +40,71 @@
|
|
|
40
40
|
"title": "ActionSpace",
|
|
41
41
|
"type": "string"
|
|
42
42
|
},
|
|
43
|
+
"BenchmarkConditionSpec": {
|
|
44
|
+
"additionalProperties": false,
|
|
45
|
+
"description": "One episode setting exactly as the author declared it — a sparse list or a cross product, with its own step\ncap, wall-clock limit and suite. {gs-plans/customization/benchmark.md \"6. 其它在 SDK 里收下、在生产里丢掉的声明\"}",
|
|
46
|
+
"properties": {
|
|
47
|
+
"max_steps": {
|
|
48
|
+
"anyOf": [
|
|
49
|
+
{
|
|
50
|
+
"exclusiveMinimum": 0,
|
|
51
|
+
"type": "integer"
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
"type": "null"
|
|
55
|
+
}
|
|
56
|
+
],
|
|
57
|
+
"default": null,
|
|
58
|
+
"title": "Max Steps"
|
|
59
|
+
},
|
|
60
|
+
"seed": {
|
|
61
|
+
"title": "Seed",
|
|
62
|
+
"type": "integer"
|
|
63
|
+
},
|
|
64
|
+
"suite": {
|
|
65
|
+
"anyOf": [
|
|
66
|
+
{
|
|
67
|
+
"type": "string"
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"type": "null"
|
|
71
|
+
}
|
|
72
|
+
],
|
|
73
|
+
"default": null,
|
|
74
|
+
"title": "Suite"
|
|
75
|
+
},
|
|
76
|
+
"task": {
|
|
77
|
+
"title": "Task",
|
|
78
|
+
"type": "string"
|
|
79
|
+
},
|
|
80
|
+
"timeout_s": {
|
|
81
|
+
"anyOf": [
|
|
82
|
+
{
|
|
83
|
+
"exclusiveMinimum": 0,
|
|
84
|
+
"type": "number"
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
"type": "null"
|
|
88
|
+
}
|
|
89
|
+
],
|
|
90
|
+
"default": null,
|
|
91
|
+
"title": "Timeout S"
|
|
92
|
+
},
|
|
93
|
+
"variant": {
|
|
94
|
+
"additionalProperties": {
|
|
95
|
+
"type": "string"
|
|
96
|
+
},
|
|
97
|
+
"title": "Variant",
|
|
98
|
+
"type": "object"
|
|
99
|
+
}
|
|
100
|
+
},
|
|
101
|
+
"required": [
|
|
102
|
+
"task",
|
|
103
|
+
"seed"
|
|
104
|
+
],
|
|
105
|
+
"title": "BenchmarkConditionSpec",
|
|
106
|
+
"type": "object"
|
|
107
|
+
},
|
|
43
108
|
"CodeBundle": {
|
|
44
109
|
"additionalProperties": false,
|
|
45
110
|
"description": "The policy's own source, shipped with the deploy so the worker can import `policy.handler_ref`\n(what Modal's automount does for the entrypoint module). A gzip'd tar of the handler's top-level module\nor package, base64 on the wire, with the sha256 of the decoded bytes so a truncated or altered upload\nis refused at the edge instead of failing as an ImportError on a GPU.",
|
|
@@ -338,6 +403,22 @@
|
|
|
338
403
|
],
|
|
339
404
|
"default": null
|
|
340
405
|
},
|
|
406
|
+
"condition_list": {
|
|
407
|
+
"anyOf": [
|
|
408
|
+
{
|
|
409
|
+
"items": {
|
|
410
|
+
"$ref": "#/$defs/BenchmarkConditionSpec"
|
|
411
|
+
},
|
|
412
|
+
"maxItems": 5000,
|
|
413
|
+
"type": "array"
|
|
414
|
+
},
|
|
415
|
+
{
|
|
416
|
+
"type": "null"
|
|
417
|
+
}
|
|
418
|
+
],
|
|
419
|
+
"default": null,
|
|
420
|
+
"title": "Condition List"
|
|
421
|
+
},
|
|
341
422
|
"conditions": {
|
|
342
423
|
"$ref": "#/$defs/Conditions"
|
|
343
424
|
},
|
|
@@ -353,6 +434,18 @@
|
|
|
353
434
|
"default": null,
|
|
354
435
|
"title": "Cpu"
|
|
355
436
|
},
|
|
437
|
+
"default_adapter_ref": {
|
|
438
|
+
"anyOf": [
|
|
439
|
+
{
|
|
440
|
+
"type": "string"
|
|
441
|
+
},
|
|
442
|
+
{
|
|
443
|
+
"type": "null"
|
|
444
|
+
}
|
|
445
|
+
],
|
|
446
|
+
"default": null,
|
|
447
|
+
"title": "Default Adapter Ref"
|
|
448
|
+
},
|
|
356
449
|
"handler_ref": {
|
|
357
450
|
"title": "Handler Ref",
|
|
358
451
|
"type": "string"
|
|
@@ -409,6 +502,19 @@
|
|
|
409
502
|
},
|
|
410
503
|
"title": "Suite Of",
|
|
411
504
|
"type": "object"
|
|
505
|
+
},
|
|
506
|
+
"timeout_s": {
|
|
507
|
+
"anyOf": [
|
|
508
|
+
{
|
|
509
|
+
"exclusiveMinimum": 0,
|
|
510
|
+
"type": "number"
|
|
511
|
+
},
|
|
512
|
+
{
|
|
513
|
+
"type": "null"
|
|
514
|
+
}
|
|
515
|
+
],
|
|
516
|
+
"default": null,
|
|
517
|
+
"title": "Timeout S"
|
|
412
518
|
}
|
|
413
519
|
},
|
|
414
520
|
"required": [
|
|
@@ -420,5 +526,5 @@
|
|
|
420
526
|
],
|
|
421
527
|
"title": "BenchmarkRegisterRequest",
|
|
422
528
|
"type": "object",
|
|
423
|
-
"x-contract-version": "0.4.
|
|
529
|
+
"x-contract-version": "0.4.5"
|
|
424
530
|
}
|
|
@@ -207,7 +207,7 @@
|
|
|
207
207
|
],
|
|
208
208
|
"title": "SeqFlags",
|
|
209
209
|
"type": "integer",
|
|
210
|
-
"x-contract-version": "0.4.
|
|
210
|
+
"x-contract-version": "0.4.5",
|
|
211
211
|
"x-flag-names": {
|
|
212
212
|
"CHECK": 65536,
|
|
213
213
|
"INFER": 16,
|
|
@@ -397,5 +397,5 @@
|
|
|
397
397
|
],
|
|
398
398
|
"title": "BenchmarkSpec",
|
|
399
399
|
"type": "object",
|
|
400
|
-
"x-contract-version": "0.4.
|
|
400
|
+
"x-contract-version": "0.4.5"
|
|
401
401
|
}
|
|
@@ -60,10 +60,35 @@
|
|
|
60
60
|
"default": null,
|
|
61
61
|
"title": "Gpus"
|
|
62
62
|
},
|
|
63
|
+
"max_usd": {
|
|
64
|
+
"anyOf": [
|
|
65
|
+
{
|
|
66
|
+
"exclusiveMinimum": 0,
|
|
67
|
+
"type": "number"
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"type": "null"
|
|
71
|
+
}
|
|
72
|
+
],
|
|
73
|
+
"default": null,
|
|
74
|
+
"title": "Max Usd"
|
|
75
|
+
},
|
|
63
76
|
"model": {
|
|
64
77
|
"title": "Model",
|
|
65
78
|
"type": "string"
|
|
66
79
|
},
|
|
80
|
+
"rerun_of": {
|
|
81
|
+
"anyOf": [
|
|
82
|
+
{
|
|
83
|
+
"type": "string"
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"type": "null"
|
|
87
|
+
}
|
|
88
|
+
],
|
|
89
|
+
"default": null,
|
|
90
|
+
"title": "Rerun Of"
|
|
91
|
+
},
|
|
67
92
|
"suite": {
|
|
68
93
|
"anyOf": [
|
|
69
94
|
{
|
|
@@ -76,6 +101,22 @@
|
|
|
76
101
|
"default": null,
|
|
77
102
|
"title": "Suite"
|
|
78
103
|
},
|
|
104
|
+
"tasks": {
|
|
105
|
+
"anyOf": [
|
|
106
|
+
{
|
|
107
|
+
"items": {
|
|
108
|
+
"type": "string"
|
|
109
|
+
},
|
|
110
|
+
"maxItems": 1000,
|
|
111
|
+
"type": "array"
|
|
112
|
+
},
|
|
113
|
+
{
|
|
114
|
+
"type": "null"
|
|
115
|
+
}
|
|
116
|
+
],
|
|
117
|
+
"default": null,
|
|
118
|
+
"title": "Tasks"
|
|
119
|
+
},
|
|
79
120
|
"trials": {
|
|
80
121
|
"anyOf": [
|
|
81
122
|
{
|
|
@@ -107,5 +148,5 @@
|
|
|
107
148
|
],
|
|
108
149
|
"title": "EvalRequest",
|
|
109
150
|
"type": "object",
|
|
110
|
-
"x-contract-version": "0.4.
|
|
151
|
+
"x-contract-version": "0.4.5"
|
|
111
152
|
}
|