sequence-base 0.8.6__tar.gz → 0.8.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sequence_base-0.8.6 → sequence_base-0.8.7}/PKG-INFO +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/pyproject.toml +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/by_name.py +8 -8
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/constants/version.py +2 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/contract/__init__.py +4 -2
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/contract/deployments.py +2 -3
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/contract/eval.py +16 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/contract/planner.py +5 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/contract/policy.py +3 -3
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ActRequest.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ActResponse.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ActionChunk.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ActionContract.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ActionStep.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/AdapterPair.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/AddLocal.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/Autoscaler.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/BenchmarkRegisterRequest.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/BenchmarkSpec.json +2 -2
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/BenchmarkStatusResponse.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/BootstrapResponse.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/Camera.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/CodeBundle.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/Concurrency.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/DeploymentAccepted.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/DeploymentPolicy.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/DeploymentRequest.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/DeploymentStatusResponse.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ErrorBody.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ErrorCodes.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/Estimate.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/EvalDryRunResponse.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/EvalReport.json +44 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/EvalRequest.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/EvalResponse.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ImageFrame.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ImageSpec.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ImageStepAddLocal.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ImageStepApt.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ImageStepEnv.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ImageStepProject.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ImageStepRun.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ImageStepTemplateFiles.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ImageStepUvPip.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/InferParam.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/Lease.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/MetricAgg.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/Observation.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ObservationContract.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ObserveRequest.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ObserveResponse.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/PlanAppend.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/PlanAppended.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/PlannerMember.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/PlannerObservation.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/PlannerObserve.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/PlannerObserved.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/PlannerPlanIn.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/PlannerPlans.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/PlannerPlansWritten.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/PlannerRobot.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/PlannerRound.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/PlannerRoundRequest.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/PlannerSpec.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/PolicyRegistration.json +2 -2
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/PolicySpec.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/Project.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/Proprioception.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/ReplaySpec.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/Resources.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/RolloutResult.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/SecretCreateRequest.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/SecretInfo.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/SecretRef.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/SeqFlags.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/SettingsChange.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/SettingsChanged.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/SettingsView.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/TaskAgg.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/TemplateFiles.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/UploadTarget.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/VolumeFile.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/VolumeInfo.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/VolumeMount.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/VolumeRef.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/VolumeUploadRequest.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/VolumeUploadResponse.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/WarmingError.json +1 -1
- {sequence_base-0.8.6 → sequence_base-0.8.7}/.gitignore +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/README.md +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/__init__.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/auth.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/codec/__init__.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/codec/blocks.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/codec/compressed.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/codec/image.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/codec/marked.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/constants/__init__.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/constants/cards.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/constants/headers.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/constants/limits.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/constants/serving.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/construct.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/contract/accounts.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/contract/adapters.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/contract/benchmarks.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/contract/connect.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/contract/errors.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/contract/flags.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/contract/observe.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/contract/own_io.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/contract/settings.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/contract/templates.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/contract/volumes.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/contract/wire.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/episode.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/errors/__init__.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/errors/transport.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/msgpack_io.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/obs/__init__.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/obs/log.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/obs/rid.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/presets/__init__.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/presets/prompts.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/presets/robot_arm.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schema_export.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/Tensors.json +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/schemas/Timeouts.json +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/telemetry/__init__.py +0 -0
- {sequence_base-0.8.6 → sequence_base-0.8.7}/sequence_base/telemetry/events.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sequence-base
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.7
|
|
4
4
|
Summary: Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness.
|
|
5
5
|
Project-URL: Changelog, https://generalsequences.com/docs/changelog/
|
|
6
6
|
Requires-Python: >=3.10
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sequence-base"
|
|
3
|
-
version = "0.8.
|
|
3
|
+
version = "0.8.7"
|
|
4
4
|
description = "Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness."
|
|
5
5
|
requires-python = ">=3.10"
|
|
6
6
|
dependencies = ["pydantic>=2.0"]
|
|
@@ -136,9 +136,9 @@ def _keys(names: list) -> str:
|
|
|
136
136
|
|
|
137
137
|
def plan_refused(plan: Any, *, infer_takes_episode: bool) -> str | None:
|
|
138
138
|
"""Why `plan`, a policy's @seq.plan method, cannot be served as it is written - or None, when it can: it is called on
|
|
139
|
-
|
|
140
|
-
the act function beside it, which must take the episode to read it.
|
|
141
|
-
"""
|
|
139
|
+
the robot's clock with the robot alone, plain or async, returning (never yielding) what is kept in the episode's state
|
|
140
|
+
for the act function beside it, which must take the episode to read it.
|
|
141
|
+
"""
|
|
142
142
|
name = getattr(plan, "__name__", "plan")
|
|
143
143
|
if inspect.isgeneratorfunction(plan) or inspect.isasyncgenfunction(plan):
|
|
144
144
|
return (f"@seq.plan {name} returns its plan rather than yielding it: return the value, which is kept in "
|
|
@@ -148,14 +148,14 @@ def plan_refused(plan: Any, *, infer_takes_episode: bool) -> str | None:
|
|
|
148
148
|
except (TypeError, ValueError):
|
|
149
149
|
params = {}
|
|
150
150
|
others = [n for n, p in params.items()
|
|
151
|
-
if n not in ("self", "
|
|
152
|
-
if "
|
|
153
|
-
return (f"@seq.plan is handed the
|
|
154
|
-
'
|
|
151
|
+
if n not in ("self", "robot") and p.default is inspect.Parameter.empty and p.kind not in _VARIADIC]
|
|
152
|
+
if "robot" not in params or others:
|
|
153
|
+
return (f"@seq.plan is handed the robot alone: write def {name}(self, robot) - read what it needs from "
|
|
154
|
+
'robot.state, and return the plan, which is kept in episode.state["seq.plans"] (None keeps the plans '
|
|
155
155
|
"as they are)")
|
|
156
156
|
if not infer_takes_episode:
|
|
157
157
|
return ("a policy with @seq.plan reads its plans where its act function runs: take episode in it and read "
|
|
158
|
-
'episode.state
|
|
158
|
+
'episode.state.get("seq.plans"), which has no plan until the first one returns')
|
|
159
159
|
return None
|
|
160
160
|
|
|
161
161
|
|
|
@@ -76,6 +76,7 @@ from sequence_base.contract.eval import (
|
|
|
76
76
|
EvalDryRunResponse,
|
|
77
77
|
EvalReport,
|
|
78
78
|
PlannerReport,
|
|
79
|
+
PlanReport,
|
|
79
80
|
EvalRequest,
|
|
80
81
|
EvalResponse,
|
|
81
82
|
EvalStatus,
|
|
@@ -130,6 +131,7 @@ from sequence_base.contract.planner import (
|
|
|
130
131
|
PLANS_MOST,
|
|
131
132
|
STATE_MOST,
|
|
132
133
|
PLAN_ON_A_POLICY,
|
|
134
|
+
PLANNERS_GONE,
|
|
133
135
|
PLANNER_ID,
|
|
134
136
|
POLICY_ID,
|
|
135
137
|
ROBOT_ID,
|
|
@@ -172,7 +174,7 @@ __all__ = [
|
|
|
172
174
|
|
|
173
175
|
"ReplaySpec", "BenchmarkSpec", "BenchmarkRegisterRequest", "BenchmarkStatusResponse",
|
|
174
176
|
"EvalRequest", "Estimate", "EvalResponse", "EvalDryRunResponse", "RolloutResult",
|
|
175
|
-
"EvalStatus", "TaskAgg", "MetricAgg", "EvalReport", "PlannerReport",
|
|
177
|
+
"EvalStatus", "TaskAgg", "MetricAgg", "EvalReport", "PlannerReport", "PlanReport",
|
|
176
178
|
|
|
177
179
|
"SecretRef", "SecretCreateRequest", "SecretInfo", "BootstrapResponse",
|
|
178
180
|
|
|
@@ -182,7 +184,7 @@ __all__ = [
|
|
|
182
184
|
"SETTINGS", "Setting", "SettingsChange", "Applied", "SettingState", "SettingsView", "SettingChanged",
|
|
183
185
|
"SettingsChanged",
|
|
184
186
|
|
|
185
|
-
"KEPT", "MOST_LAYERS", "PLANS_MOST", "STATE_MOST", "PLAN_ON_A_POLICY", "PLANNER_ID", "POLICY_ID", "ROBOT_ID", "WAITING_FOR_PLAN",
|
|
187
|
+
"KEPT", "MOST_LAYERS", "PLANS_MOST", "STATE_MOST", "PLAN_ON_A_POLICY", "PLANNERS_GONE", "PLANNER_ID", "POLICY_ID", "ROBOT_ID", "WAITING_FOR_PLAN",
|
|
186
188
|
"PlannerSpec",
|
|
187
189
|
"PlannerRoundRequest", "PlannerRobot", "PlannerMember", "PlannerObservation", "PlannerPlanIn", "PlannerRound",
|
|
188
190
|
"PlannerPlans", "PlannerPlansWritten", "PlanAppend", "PlanAppended", "PlannerObserve", "PlannerObserved",
|
|
@@ -21,7 +21,7 @@ from typing import Any, Literal
|
|
|
21
21
|
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
22
22
|
|
|
23
23
|
from sequence_base.contract.accounts import SecretRef
|
|
24
|
-
from sequence_base.contract.planner import
|
|
24
|
+
from sequence_base.contract.planner import PlannerSpec
|
|
25
25
|
from sequence_base.contract.policy import (
|
|
26
26
|
Concurrency,
|
|
27
27
|
Episodes,
|
|
@@ -204,6 +204,7 @@ class DeploymentPolicy(BaseModel):
|
|
|
204
204
|
models: list[str] | None = Field(default=None, max_length=16)
|
|
205
205
|
|
|
206
206
|
|
|
207
|
+
|
|
207
208
|
plan_every_s: float | None = Field(default=None, gt=0)
|
|
208
209
|
|
|
209
210
|
|
|
@@ -219,8 +220,6 @@ class DeploymentPolicy(BaseModel):
|
|
|
219
220
|
raise ValueError(f"a planner answers no act, so it declares no {', '.join(said)}")
|
|
220
221
|
if self.plan_every_s is None:
|
|
221
222
|
raise ValueError("a planner plans: its policy block says plan_every_s")
|
|
222
|
-
elif self.plan_every_s is not None:
|
|
223
|
-
raise ValueError(PLAN_ON_A_POLICY)
|
|
224
223
|
return self
|
|
225
224
|
|
|
226
225
|
@model_validator(mode="after")
|
|
@@ -34,6 +34,7 @@ __all__ = [
|
|
|
34
34
|
"MetricAgg",
|
|
35
35
|
"EvalReport",
|
|
36
36
|
"PlannerReport",
|
|
37
|
+
"PlanReport",
|
|
37
38
|
]
|
|
38
39
|
|
|
39
40
|
|
|
@@ -352,6 +353,19 @@ class PlannerReport(BaseModel):
|
|
|
352
353
|
model_calls_usd: float | None = None
|
|
353
354
|
|
|
354
355
|
|
|
356
|
+
class PlanReport(BaseModel):
|
|
357
|
+
"""An eval of a policy with @seq.plan: how often it planned each episode on the robot's clock, the plans it kept over
|
|
358
|
+
every episode (each in its episode's record and replay), and what the model calls of this eval cost, billed as
|
|
359
|
+
requests beside the machines."""
|
|
360
|
+
|
|
361
|
+
model_config = ConfigDict(extra="forbid")
|
|
362
|
+
|
|
363
|
+
every_s: float
|
|
364
|
+
plans: int = 0
|
|
365
|
+
|
|
366
|
+
model_calls_usd: float | None = None
|
|
367
|
+
|
|
368
|
+
|
|
355
369
|
class EvalReport(BaseModel):
|
|
356
370
|
"""GET /v1/eval/{id} (eval-raw revision). NO hard-coded `overall_sr` / `success_rate` top-level field —
|
|
357
371
|
`metrics` is a passthrough dict keyed by the user's metric names, each value a `MetricAgg`; `success` is among
|
|
@@ -387,3 +401,5 @@ class EvalReport(BaseModel):
|
|
|
387
401
|
arm_params: list[ArmParams] | None = None
|
|
388
402
|
|
|
389
403
|
planner: PlannerReport | None = None
|
|
404
|
+
|
|
405
|
+
plan: PlanReport | None = None
|
|
@@ -59,6 +59,11 @@ PLAN_ON_A_POLICY = ("@seq.plan is a planner's now: a policy is planned by a @seq
|
|
|
59
59
|
"episode.state[\"seq.plans\"][-1][\"value\"]")
|
|
60
60
|
|
|
61
61
|
|
|
62
|
+
PLANNERS_GONE = ("@seq.planner is gone: planning is part of the policy now. Put a @seq.plan method in your @seq.policy "
|
|
63
|
+
"class - def plan(self, robot), returning the next step - and connect to the policy by its name, "
|
|
64
|
+
"connect(\"<policy>\").")
|
|
65
|
+
|
|
66
|
+
|
|
62
67
|
def check_name(value: str, what: str) -> str:
|
|
63
68
|
"""`value` as a planner's or a member's name, else ValueError saying the rule."""
|
|
64
69
|
if not isinstance(value, str) or not NAME.fullmatch(value):
|
|
@@ -16,7 +16,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_valida
|
|
|
16
16
|
|
|
17
17
|
from sequence_base.contract.accounts import SecretRef
|
|
18
18
|
from sequence_base.contract.flags import SeqFlags
|
|
19
|
-
from sequence_base.contract.planner import
|
|
19
|
+
from sequence_base.contract.planner import PlannerSpec
|
|
20
20
|
|
|
21
21
|
__all__ = [
|
|
22
22
|
"Episodes",
|
|
@@ -544,8 +544,8 @@ class PolicyRegistration(BaseModel):
|
|
|
544
544
|
f"not {', '.join(other)}")
|
|
545
545
|
if SeqFlags.PLAN not in self.bindings or self.plan_every_s is None:
|
|
546
546
|
raise ValueError("a planner plans: it binds one @seq.plan(every_s=...)")
|
|
547
|
-
elif SeqFlags.PLAN in self.bindings
|
|
548
|
-
raise ValueError(
|
|
547
|
+
elif (SeqFlags.PLAN in self.bindings) != (self.plan_every_s is not None):
|
|
548
|
+
raise ValueError("a policy's @seq.plan says how often it plans: it binds one @seq.plan(every_s=...)")
|
|
549
549
|
return self
|
|
550
550
|
|
|
551
551
|
@model_validator(mode="after")
|
|
@@ -435,7 +435,7 @@
|
|
|
435
435
|
],
|
|
436
436
|
"title": "SeqFlags",
|
|
437
437
|
"type": "integer",
|
|
438
|
-
"x-contract-version": "0.5.
|
|
438
|
+
"x-contract-version": "0.5.47",
|
|
439
439
|
"x-flag-names": {
|
|
440
440
|
"ACTION": 1048576,
|
|
441
441
|
"CHECK": 65536,
|
|
@@ -689,5 +689,5 @@
|
|
|
689
689
|
],
|
|
690
690
|
"title": "BenchmarkSpec",
|
|
691
691
|
"type": "object",
|
|
692
|
-
"x-contract-version": "0.5.
|
|
692
|
+
"x-contract-version": "0.5.47"
|
|
693
693
|
}
|
|
@@ -269,6 +269,38 @@
|
|
|
269
269
|
"title": "ParamUsed",
|
|
270
270
|
"type": "object"
|
|
271
271
|
},
|
|
272
|
+
"PlanReport": {
|
|
273
|
+
"additionalProperties": false,
|
|
274
|
+
"description": "An eval of a policy with @seq.plan: how often it planned each episode on the robot's clock, the plans it kept over\nevery episode (each in its episode's record and replay), and what the model calls of this eval cost, billed as\nrequests beside the machines.",
|
|
275
|
+
"properties": {
|
|
276
|
+
"every_s": {
|
|
277
|
+
"title": "Every S",
|
|
278
|
+
"type": "number"
|
|
279
|
+
},
|
|
280
|
+
"model_calls_usd": {
|
|
281
|
+
"anyOf": [
|
|
282
|
+
{
|
|
283
|
+
"type": "number"
|
|
284
|
+
},
|
|
285
|
+
{
|
|
286
|
+
"type": "null"
|
|
287
|
+
}
|
|
288
|
+
],
|
|
289
|
+
"default": null,
|
|
290
|
+
"title": "Model Calls Usd"
|
|
291
|
+
},
|
|
292
|
+
"plans": {
|
|
293
|
+
"default": 0,
|
|
294
|
+
"title": "Plans",
|
|
295
|
+
"type": "integer"
|
|
296
|
+
}
|
|
297
|
+
},
|
|
298
|
+
"required": [
|
|
299
|
+
"every_s"
|
|
300
|
+
],
|
|
301
|
+
"title": "PlanReport",
|
|
302
|
+
"type": "object"
|
|
303
|
+
},
|
|
272
304
|
"PlannerReport": {
|
|
273
305
|
"additionalProperties": false,
|
|
274
306
|
"description": "An eval through a planner (EvalRequest.model a planner): the planner it ran through and what it did - how often it\nplanned each episode on the robot's clock, the plans it wrote over every episode (each kept in its episode's record\nand replay), and what its model calls cost, billed as requests beside the machines.",
|
|
@@ -406,6 +438,17 @@
|
|
|
406
438
|
"title": "Metrics",
|
|
407
439
|
"type": "object"
|
|
408
440
|
},
|
|
441
|
+
"plan": {
|
|
442
|
+
"anyOf": [
|
|
443
|
+
{
|
|
444
|
+
"$ref": "#/$defs/PlanReport"
|
|
445
|
+
},
|
|
446
|
+
{
|
|
447
|
+
"type": "null"
|
|
448
|
+
}
|
|
449
|
+
],
|
|
450
|
+
"default": null
|
|
451
|
+
},
|
|
409
452
|
"planner": {
|
|
410
453
|
"anyOf": [
|
|
411
454
|
{
|
|
@@ -470,5 +513,5 @@
|
|
|
470
513
|
],
|
|
471
514
|
"title": "EvalReport",
|
|
472
515
|
"type": "object",
|
|
473
|
-
"x-contract-version": "0.5.
|
|
516
|
+
"x-contract-version": "0.5.47"
|
|
474
517
|
}
|
|
@@ -654,7 +654,7 @@
|
|
|
654
654
|
],
|
|
655
655
|
"title": "SeqFlags",
|
|
656
656
|
"type": "integer",
|
|
657
|
-
"x-contract-version": "0.5.
|
|
657
|
+
"x-contract-version": "0.5.47",
|
|
658
658
|
"x-flag-names": {
|
|
659
659
|
"ACTION": 1048576,
|
|
660
660
|
"CHECK": 65536,
|
|
@@ -769,5 +769,5 @@
|
|
|
769
769
|
],
|
|
770
770
|
"title": "PolicyRegistration",
|
|
771
771
|
"type": "object",
|
|
772
|
-
"x-contract-version": "0.5.
|
|
772
|
+
"x-contract-version": "0.5.47"
|
|
773
773
|
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|