sequence-base 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sequence_base/__init__.py +134 -0
- sequence_base/auth.py +35 -0
- sequence_base/codec/__init__.py +25 -0
- sequence_base/codec/image.py +136 -0
- sequence_base/codec/resize.py +147 -0
- sequence_base/constants/__init__.py +0 -0
- sequence_base/constants/env.py +14 -0
- sequence_base/constants/headers.py +5 -0
- sequence_base/constants/version.py +9 -0
- sequence_base/contract/__init__.py +111 -0
- sequence_base/contract/accounts.py +93 -0
- sequence_base/contract/act.py +89 -0
- sequence_base/contract/benchmarks.py +130 -0
- sequence_base/contract/connect.py +53 -0
- sequence_base/contract/deployments.py +197 -0
- sequence_base/contract/errors.py +45 -0
- sequence_base/contract/eval.py +201 -0
- sequence_base/contract/flags.py +92 -0
- sequence_base/contract/policy.py +172 -0
- sequence_base/errors/__init__.py +0 -0
- sequence_base/errors/transport.py +36 -0
- sequence_base/obs/__init__.py +0 -0
- sequence_base/obs/log.py +57 -0
- sequence_base/obs/rid.py +31 -0
- sequence_base/schema_export.py +238 -0
- sequence_base/schemas/ActionChunk.json +102 -0
- sequence_base/schemas/ActionContract.json +43 -0
- sequence_base/schemas/ActionStep.json +23 -0
- sequence_base/schemas/AdapterTransform.json +42 -0
- sequence_base/schemas/Autoscaler.json +43 -0
- sequence_base/schemas/BenchmarkRegisterRequest.json +302 -0
- sequence_base/schemas/BenchmarkSpec.json +401 -0
- sequence_base/schemas/BootstrapResponse.json +76 -0
- sequence_base/schemas/ClampInfo.json +27 -0
- sequence_base/schemas/CodeBundle.json +27 -0
- sequence_base/schemas/Concurrency.json +21 -0
- sequence_base/schemas/Conditions.json +59 -0
- sequence_base/schemas/ConnectRequest.json +16 -0
- sequence_base/schemas/ConnectResponse.json +81 -0
- sequence_base/schemas/DeploymentAccepted.json +22 -0
- sequence_base/schemas/DeploymentPolicy.json +101 -0
- sequence_base/schemas/DeploymentRequest.json +467 -0
- sequence_base/schemas/DeploymentStatusResponse.json +45 -0
- sequence_base/schemas/ErrorBody.json +32 -0
- sequence_base/schemas/Estimate.json +25 -0
- sequence_base/schemas/EvalDryRunResponse.json +41 -0
- sequence_base/schemas/EvalReport.json +179 -0
- sequence_base/schemas/EvalRequest.json +111 -0
- sequence_base/schemas/EvalResponse.json +88 -0
- sequence_base/schemas/ImageFrame.json +36 -0
- sequence_base/schemas/ImageSpec.json +123 -0
- sequence_base/schemas/ImageStepApt.json +18 -0
- sequence_base/schemas/ImageStepEnv.json +18 -0
- sequence_base/schemas/ImageStepRun.json +18 -0
- sequence_base/schemas/ImageStepUvPip.json +18 -0
- sequence_base/schemas/Lease.json +28 -0
- sequence_base/schemas/LossyTransform.json +21 -0
- sequence_base/schemas/MetricAgg.json +87 -0
- sequence_base/schemas/Observation.json +133 -0
- sequence_base/schemas/PolicyRegistration.json +438 -0
- sequence_base/schemas/PolicySpec.json +346 -0
- sequence_base/schemas/Proprioception.json +67 -0
- sequence_base/schemas/RenderSpec.json +22 -0
- sequence_base/schemas/Resources.json +29 -0
- sequence_base/schemas/RolloutResult.json +59 -0
- sequence_base/schemas/SecretCreateRequest.json +25 -0
- sequence_base/schemas/SecretInfo.json +44 -0
- sequence_base/schemas/SecretRef.json +16 -0
- sequence_base/schemas/SeqFlags.json +33 -0
- sequence_base/schemas/Step.json +68 -0
- sequence_base/schemas/SuiteAgg.json +26 -0
- sequence_base/schemas/TaskAgg.json +26 -0
- sequence_base/schemas/Timeouts.json +20 -0
- sequence_base/schemas/UsageReport.json +38 -0
- sequence_base/schemas/Variant.json +24 -0
- sequence_base/schemas/VolumeAttach.json +26 -0
- sequence_base/schemas/VolumeMount.json +21 -0
- sequence_base/schemas/VolumeRef.json +28 -0
- sequence_base/schemas/WarmingError.json +50 -0
- sequence_base/telemetry/__init__.py +13 -0
- sequence_base/telemetry/events.py +101 -0
- sequence_base-0.3.0.dist-info/METADATA +12 -0
- sequence_base-0.3.0.dist-info/RECORD +84 -0
- sequence_base-0.3.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""
|
|
2
|
+
sequence-base — the shared foundation for the Sequences platform.
|
|
3
|
+
|
|
4
|
+
The wire protocol + primitives that the robot SDK, the gateway, and the container harness ALL use,
|
|
5
|
+
defined once so they cannot drift. Import from here; never redefine.
|
|
6
|
+
|
|
7
|
+
from sequence_base import Observation, ActionChunk # contract
|
|
8
|
+
from sequence_base import log, new_rid, HEADER # correlation
|
|
9
|
+
from sequence_base import TransportWarming, TransportOverloaded # error taxonomy
|
|
10
|
+
|
|
11
|
+
Only shared-by-≥2-and-drifts-if-duplicated things live here. Model-specific, provider-specific, and
|
|
12
|
+
catalogue values stay in container / gateway / spec.py.
|
|
13
|
+
"""
|
|
14
|
+
# contract — the full shared wire contract (obs/action + registry + deployments + connect + eval +
|
|
15
|
+
# accounts + error bodies). One definition per shape; see sequence_base/contract for the grouping.
|
|
16
|
+
from sequence_base.contract import ( # noqa: F401 (re-export surface)
|
|
17
|
+
MAX_CODE_BUNDLE_BYTES,
|
|
18
|
+
ActionChunk,
|
|
19
|
+
ActionContract,
|
|
20
|
+
ActionSpace,
|
|
21
|
+
ActionStep,
|
|
22
|
+
AdapterKind,
|
|
23
|
+
AdapterTransform,
|
|
24
|
+
Autoscaler,
|
|
25
|
+
BenchmarkRegisterRequest,
|
|
26
|
+
BenchmarkSpec,
|
|
27
|
+
BootstrapResponse,
|
|
28
|
+
ClampInfo,
|
|
29
|
+
CodeBundle,
|
|
30
|
+
Concurrency,
|
|
31
|
+
Conditions,
|
|
32
|
+
ConnectRequest,
|
|
33
|
+
ConnectResponse,
|
|
34
|
+
DeploymentAccepted,
|
|
35
|
+
DeploymentPolicy,
|
|
36
|
+
DeploymentRequest,
|
|
37
|
+
DeploymentState,
|
|
38
|
+
DeploymentStatusResponse,
|
|
39
|
+
ErrorBody,
|
|
40
|
+
ErrorCode,
|
|
41
|
+
Estimate,
|
|
42
|
+
EvalDryRunResponse,
|
|
43
|
+
EvalReport,
|
|
44
|
+
EvalRequest,
|
|
45
|
+
EvalResponse,
|
|
46
|
+
EvalStatus,
|
|
47
|
+
ImageFrame,
|
|
48
|
+
ImageSpec,
|
|
49
|
+
ImageStep,
|
|
50
|
+
ImageStepApt,
|
|
51
|
+
ImageStepEnv,
|
|
52
|
+
ImageStepRun,
|
|
53
|
+
ImageStepUvPip,
|
|
54
|
+
Lease,
|
|
55
|
+
LossyTransform,
|
|
56
|
+
MetricAgg,
|
|
57
|
+
Observation,
|
|
58
|
+
PolicyRegistration,
|
|
59
|
+
PolicySpec,
|
|
60
|
+
Proprioception,
|
|
61
|
+
RenderSpec,
|
|
62
|
+
Resources,
|
|
63
|
+
RolloutResult,
|
|
64
|
+
SecretCreateRequest,
|
|
65
|
+
SecretInfo,
|
|
66
|
+
SecretRef,
|
|
67
|
+
SeqFlags,
|
|
68
|
+
Step,
|
|
69
|
+
SuiteAgg,
|
|
70
|
+
TaskAgg,
|
|
71
|
+
Timeouts,
|
|
72
|
+
UsageReport,
|
|
73
|
+
Variant,
|
|
74
|
+
VolumeAttach,
|
|
75
|
+
VolumeMount,
|
|
76
|
+
VolumeRef,
|
|
77
|
+
WarmingError,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
# the single schema-level version stamp (embedded in every exported JSON Schema)
|
|
81
|
+
from sequence_base.constants.version import CONTRACT_VERSION
|
|
82
|
+
|
|
83
|
+
# obs — correlation id + structured logging (all three sides)
|
|
84
|
+
from sequence_base.obs.log import Timer, enabled, log
|
|
85
|
+
from sequence_base.obs.rid import HEADER, clean, new_rid
|
|
86
|
+
|
|
87
|
+
# errors — shared transport status taxonomy (gateway + container)
|
|
88
|
+
from sequence_base.errors.transport import (
|
|
89
|
+
TransportOverloaded,
|
|
90
|
+
TransportRejected,
|
|
91
|
+
TransportUnavailable,
|
|
92
|
+
TransportWarming,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
# auth — Bearer secret helpers (gateway direct-connect / internal; container guard; worker emit)
|
|
96
|
+
from sequence_base import auth
|
|
97
|
+
|
|
98
|
+
# telemetry — cold-start event schema (worker emits, gateway ingests)
|
|
99
|
+
from sequence_base.telemetry.events import WorkerLogEvent
|
|
100
|
+
|
|
101
|
+
# NOTE: codec (image encode/decode) is intentionally NOT re-exported here — it needs PIL + numpy
|
|
102
|
+
# (the [codec] extra), and the gateway installs the light core. Import it explicitly where needed:
|
|
103
|
+
# from sequence_base.codec import encode, decode, ResizeSpec
|
|
104
|
+
|
|
105
|
+
__all__ = [
|
|
106
|
+
# contract 4 — /act data plane
|
|
107
|
+
"ImageFrame", "Proprioception", "Observation", "ActionSpace", "ActionStep", "ActionChunk",
|
|
108
|
+
# contract 1 — flags + policy registry
|
|
109
|
+
"SeqFlags", "ImageStepUvPip", "ImageStepApt", "ImageStepEnv", "ImageStepRun", "ImageStep", "ImageSpec",
|
|
110
|
+
"VolumeMount", "VolumeRef", "Concurrency", "ActionContract", "PolicySpec", "PolicyRegistration",
|
|
111
|
+
# contract 2 — deployments
|
|
112
|
+
"MAX_CODE_BUNDLE_BYTES", "CodeBundle",
|
|
113
|
+
"Resources", "Autoscaler", "Timeouts", "VolumeAttach", "DeploymentPolicy", "DeploymentRequest",
|
|
114
|
+
"DeploymentAccepted", "DeploymentState", "DeploymentStatusResponse",
|
|
115
|
+
# contract 3 — connect / lease
|
|
116
|
+
"Lease", "ConnectRequest", "ConnectResponse",
|
|
117
|
+
# contract 5 — benchmark + eval
|
|
118
|
+
"Variant", "Conditions", "RenderSpec", "Step", "BenchmarkSpec", "BenchmarkRegisterRequest",
|
|
119
|
+
"EvalRequest", "ClampInfo", "Estimate", "EvalResponse", "EvalDryRunResponse", "RolloutResult",
|
|
120
|
+
"EvalStatus", "SuiteAgg", "TaskAgg", "MetricAgg", "LossyTransform", "EvalReport",
|
|
121
|
+
"AdapterKind", "AdapterTransform",
|
|
122
|
+
# contract 6 — accounts / secret / billing
|
|
123
|
+
"SecretRef", "SecretCreateRequest", "SecretInfo", "BootstrapResponse", "UsageReport",
|
|
124
|
+
# error bodies + version
|
|
125
|
+
"ErrorCode", "ErrorBody", "WarmingError", "CONTRACT_VERSION",
|
|
126
|
+
# obs
|
|
127
|
+
"HEADER", "new_rid", "clean", "log", "Timer", "enabled",
|
|
128
|
+
# errors (exception taxonomy)
|
|
129
|
+
"TransportUnavailable", "TransportWarming", "TransportRejected", "TransportOverloaded",
|
|
130
|
+
# auth + telemetry
|
|
131
|
+
"auth", "WorkerLogEvent",
|
|
132
|
+
]
|
|
133
|
+
|
|
134
|
+
__version__ = "0.3.0"
|
sequence_base/auth.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Bearer-token auth helpers — one implementation of the direct-connect / internal secret check.
|
|
3
|
+
|
|
4
|
+
Three consumers, one rule: the container's inbound guard on a directly-exposed pod's /act, the
|
|
5
|
+
gateway's SEQ_DIRECT_SECRET issuance + its /v1/internal/worker-log secret check, and the worker's
|
|
6
|
+
emit() which builds the outbound header. compare_digest, not `==`, everywhere — a wrong secret must
|
|
7
|
+
not be recoverable from response timing.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hmac
|
|
12
|
+
|
|
13
|
+
from sequence_base.constants.headers import AUTH_SCHEME
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def bearer(secret: str) -> str:
|
|
17
|
+
"""The `Authorization` header value carrying `secret` (`"Bearer <secret>"`)."""
|
|
18
|
+
return f"{AUTH_SCHEME} {secret}"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def verify(header_value: str | None, secret: str) -> bool:
|
|
22
|
+
"""Constant-time check that an incoming `Authorization` header carries `secret`.
|
|
23
|
+
|
|
24
|
+
`compare_digest`, not `==`, so a wrong secret cannot be recovered from response timing. A missing
|
|
25
|
+
header compares against the expected value and fails, without a length-based early-out.
|
|
26
|
+
"""
|
|
27
|
+
return hmac.compare_digest(header_value or "", bearer(secret))
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def parse_bearer(header_value: str | None) -> str | None:
|
|
31
|
+
"""The token from a `Bearer <token>` header, or None if the scheme prefix is absent."""
|
|
32
|
+
prefix = AUTH_SCHEME + " "
|
|
33
|
+
if header_value and header_value.startswith(prefix):
|
|
34
|
+
return header_value[len(prefix):]
|
|
35
|
+
return None
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Image codec — the frame <-> wire transform, shared so the SDK (encode) and the worker (decode)
|
|
3
|
+
cannot drift on the wire format, and so the SDK (pre-size) and the gateway (validate geometry) share
|
|
4
|
+
one `resize_with_pad` and one aspect band.
|
|
5
|
+
|
|
6
|
+
Needs Pillow + numpy: `pip install 'sequence-base[codec]'`. The gateway, which neither encodes nor
|
|
7
|
+
decodes, installs the light core and never imports this.
|
|
8
|
+
|
|
9
|
+
from sequence_base.codec import encode, decode, ResizeSpec
|
|
10
|
+
"""
|
|
11
|
+
from sequence_base.codec.image import DEFAULT_JPEG_QUALITY, decode, encode
|
|
12
|
+
from sequence_base.codec.resize import (
|
|
13
|
+
ASPECT_TOLERANCE_PX,
|
|
14
|
+
MAX_UPLOAD_EDGE,
|
|
15
|
+
ResizeSpec,
|
|
16
|
+
ratio_name,
|
|
17
|
+
refuse_wrong_aspect,
|
|
18
|
+
resize_with_pad,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"encode", "decode", "DEFAULT_JPEG_QUALITY",
|
|
23
|
+
"ResizeSpec", "resize_with_pad", "refuse_wrong_aspect", "ratio_name",
|
|
24
|
+
"MAX_UPLOAD_EDGE", "ASPECT_TOLERANCE_PX",
|
|
25
|
+
]
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The image codec: one camera frame <-> the base64 string the wire carries.
|
|
3
|
+
|
|
4
|
+
`encode` (frame -> base64 JPEG) is the SDK's client-side path; `decode` (base64 -> uint8 RGB array)
|
|
5
|
+
is the container/worker's server-side path. They live together so the wire format — JPEG, quality
|
|
6
|
+
95, RGB — is defined once and provable by a round-trip test, and so the SDK and the worker cannot
|
|
7
|
+
drift on what they put on and take off the wire.
|
|
8
|
+
|
|
9
|
+
Pillow is imported lazily inside each function, never at module scope: a caller who only passes
|
|
10
|
+
pre-encoded strings must not need it installed.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import base64
|
|
15
|
+
import io
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from .resize import MAX_UPLOAD_EDGE, ResizeSpec, filters, refuse_wrong_aspect, resize_with_pad
|
|
19
|
+
|
|
20
|
+
# Quality 95, and the number is measured rather than chosen for looking safe. On a real DROID frame
|
|
21
|
+
# with the sampling noise pinned, the encoding moves the resulting action by: q95 0.51% of action
|
|
22
|
+
# amplitude, q90 0.20%, q85 1.86%, q75 2.86% — 85 and below is where the error jumps by an order of
|
|
23
|
+
# magnitude, that is the line. 95 rather than the empirically-best 90 because the effect is not
|
|
24
|
+
# monotonic in quality on a single frame, and tuning to one frame is fitting noise.
|
|
25
|
+
# {sequences-models models/pi05_droid/handler.py "Q95 MAX|DELTA| 0.0048 0.51% ... Q85 1.86%"}
|
|
26
|
+
DEFAULT_JPEG_QUALITY = 95
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def encode(frame: Any, spec: ResizeSpec | None = None, *, quality: int = DEFAULT_JPEG_QUALITY) -> str:
|
|
30
|
+
"""
|
|
31
|
+
One camera frame to the base64 image that `ImageFrame.data` carries.
|
|
32
|
+
|
|
33
|
+
`frame` may be a numpy array, a PIL image, raw `bytes` already encoded, or a `str` already base64
|
|
34
|
+
— the last two pass through untouched, so a caller with their own pipeline is not forced through
|
|
35
|
+
this one.
|
|
36
|
+
|
|
37
|
+
`spec` (a `ResizeSpec`) carries the model's geometry. When it declares a training aspect ratio, a
|
|
38
|
+
frame outside it is REFUSED — see `refuse_wrong_aspect`, not quietly reshaped. With no spec (or no
|
|
39
|
+
`client_resize`) the frame falls to `MAX_UPLOAD_EDGE`. Codec is JPEG at `quality`, unconditionally
|
|
40
|
+
— every served handler asks for JPEG in as many words, so there is no size probe and no second
|
|
41
|
+
encode. {handlers pi05_droid/cosmos3/groot all "SEND BASE64 JPEG"}
|
|
42
|
+
"""
|
|
43
|
+
if isinstance(frame, str):
|
|
44
|
+
return frame
|
|
45
|
+
if isinstance(frame, (bytes, bytearray)):
|
|
46
|
+
return base64.b64encode(bytes(frame)).decode()
|
|
47
|
+
|
|
48
|
+
try:
|
|
49
|
+
from PIL import Image
|
|
50
|
+
except ImportError as e: # pragma: no cover - depends on the install
|
|
51
|
+
raise ImportError(
|
|
52
|
+
"encoding an array or PIL image needs Pillow: pip install 'sequence-base[codec]'. "
|
|
53
|
+
"Pass an already-encoded base64 str or bytes to avoid it."
|
|
54
|
+
) from e
|
|
55
|
+
|
|
56
|
+
img = frame if hasattr(frame, "save") else Image.fromarray(frame)
|
|
57
|
+
if img.mode != "RGB":
|
|
58
|
+
img = img.convert("RGB")
|
|
59
|
+
|
|
60
|
+
# Downscale before encoding. Every model resizes the frame to its own input size anyway, so pixels
|
|
61
|
+
# above that only cost transport — and past a point they do not cost, they fail (a body over ~2 MB
|
|
62
|
+
# is dropped by the worker queue with no error and no job id, surfacing as a hang). The target and
|
|
63
|
+
# the filter both come from the spec; neither is derived here. `client_resize=None` means the
|
|
64
|
+
# model declined pre-resizing and is not a gap: it falls to the cap below, as an unknown model
|
|
65
|
+
# does. HOW the frame reaches that size comes from `input_resize` ("pad" = the server's own
|
|
66
|
+
# resize_with_pad, "stretch" = unconditional interpolate).
|
|
67
|
+
want = None
|
|
68
|
+
cr = spec.client_resize if spec else None
|
|
69
|
+
if cr:
|
|
70
|
+
# `shortest_edge` scales the short side to n and lets the ratio follow — the only way to
|
|
71
|
+
# express a server step defined as "SmallestMaxSize(256)" (1280x720 and 1920x1080 both to
|
|
72
|
+
# 455x256), which no fixed [w, h] covers. round(), matching albumentations. `size` is a fixed
|
|
73
|
+
# target.
|
|
74
|
+
if "shortest_edge" in cr:
|
|
75
|
+
# Shrink only. SmallestMaxSize also UPSCALES a frame whose short edge is under the target,
|
|
76
|
+
# and reproducing that here is strictly worse: the payload grows and the upscale happens
|
|
77
|
+
# in PIL when the server was going to do it in cv2 anyway. Leaving a small frame alone
|
|
78
|
+
# hands the whole operation to the server, where it is exact.
|
|
79
|
+
n = int(cr["shortest_edge"])
|
|
80
|
+
if min(img.size) > n:
|
|
81
|
+
scale = n / min(img.size)
|
|
82
|
+
want = (max(1, round(img.width * scale)), max(1, round(img.height * scale)))
|
|
83
|
+
else:
|
|
84
|
+
want = img.size
|
|
85
|
+
else:
|
|
86
|
+
want = (int(cr["size"][0]), int(cr["size"][1]))
|
|
87
|
+
if img.size != want:
|
|
88
|
+
# The filter is named by the server, not chosen here: this resize stands in for one the
|
|
89
|
+
# model's own transform would otherwise run, so it has to be the SAME operation. An
|
|
90
|
+
# unrecognised name raises rather than falling back — a silent default is how the LANCZOS
|
|
91
|
+
# bug shipped. {MEASURED: BILINEAR to pi05's content block is 0.000/255, LANCZOS 0.779/15}
|
|
92
|
+
try:
|
|
93
|
+
resample = filters()[cr["filter"]]
|
|
94
|
+
except KeyError:
|
|
95
|
+
raise ValueError(
|
|
96
|
+
f"{spec.label} asks for resize filter {cr['filter']!r}, which this version of "
|
|
97
|
+
f"sequence-base does not implement (has: {sorted(filters())}). Upgrade, or pass "
|
|
98
|
+
f"an already-encoded frame to bypass this step."
|
|
99
|
+
) from None
|
|
100
|
+
# Refused before either branch, not inside "pad": a stretch model distorts a wrong ratio
|
|
101
|
+
# instead of letterboxing it, which is worse rather than exempt.
|
|
102
|
+
refuse_wrong_aspect(img, spec.aspect_ratio, max(want), spec.label)
|
|
103
|
+
if "shortest_edge" in cr:
|
|
104
|
+
# Already proportional, so nothing to letterbox — and letterboxing would actively
|
|
105
|
+
# damage it (a spurious black row the server carries into its crop as if it were scene).
|
|
106
|
+
img = img.resize(want, resample)
|
|
107
|
+
elif (spec.input_resize or "pad") == "pad":
|
|
108
|
+
img = resize_with_pad(img, want[0], want[1], resample)
|
|
109
|
+
else:
|
|
110
|
+
img = img.resize(want, resample)
|
|
111
|
+
elif max(img.size) > MAX_UPLOAD_EDGE:
|
|
112
|
+
scale = MAX_UPLOAD_EDGE / max(img.size)
|
|
113
|
+
# round() not int(): int() truncates a 640.9 on one axis and 359.9 on the other, drifting the
|
|
114
|
+
# aspect ratio by up to half a percent for no reason.
|
|
115
|
+
img = img.resize((max(1, round(img.width * scale)),
|
|
116
|
+
max(1, round(img.height * scale))), Image.LANCZOS)
|
|
117
|
+
|
|
118
|
+
buf = io.BytesIO()
|
|
119
|
+
img.save(buf, format="JPEG", quality=quality)
|
|
120
|
+
return base64.b64encode(buf.getvalue()).decode()
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def decode(data: str) -> "Any":
|
|
124
|
+
"""
|
|
125
|
+
base64 JPEG/PNG -> HxWx3 uint8 RGB array — the worker's server-side inverse of `encode`.
|
|
126
|
+
|
|
127
|
+
This is what turns `ImageFrame.data` back into pixels for the model. Send JPEG q95: 16x smaller
|
|
128
|
+
than a JSON int array, ~0.5% action delta. {sequences-models worker handler decode_image}
|
|
129
|
+
|
|
130
|
+
Returns a numpy array; numpy is part of the `[codec]` extra alongside Pillow.
|
|
131
|
+
"""
|
|
132
|
+
import numpy as np
|
|
133
|
+
from PIL import Image
|
|
134
|
+
|
|
135
|
+
raw = base64.b64decode(data, validate=False)
|
|
136
|
+
return np.asarray(Image.open(io.BytesIO(raw)).convert("RGB"), dtype=np.uint8)
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Image resize geometry — the part of the codec that must match the SERVER's own transform, so a
|
|
3
|
+
frame pre-sized by the SDK and one sent raw reach the model as identical pixels.
|
|
4
|
+
|
|
5
|
+
Extracted verbatim from the SDK's `_core/encode.py`, de-coupled from its `Model` catalogue type:
|
|
6
|
+
`encode` here takes a `ResizeSpec` value object (the geometry three-tuple + a label for errors)
|
|
7
|
+
instead of a `Model`, so the gateway (which validates geometry) and the SDK (which pre-sizes) share
|
|
8
|
+
one implementation of `resize_with_pad` and the aspect band.
|
|
9
|
+
|
|
10
|
+
Pillow is imported lazily, never at module scope — a caller who only passes pre-encoded strings must
|
|
11
|
+
not need it installed (the SDK's single-dependency promise on a Jetson's pinned environment).
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
# The longest edge `encode` uploads when no client_resize is declared. See image.encode for why a
|
|
19
|
+
# cap exists and why this number: 640 is above every served input size (pi05 224x126, GR00T
|
|
20
|
+
# 256x256, cosmos3 640x360/view), so it discards nothing any model reads, and it keeps a phone-sized
|
|
21
|
+
# frame from silently hanging the worker queue (a body over ~2 MB is dropped with no error, no job
|
|
22
|
+
# id). {sequences-models cosmos3_edge_droid/handler.py "A 2.37 MB BODY IS REJECTED BY `/RUN` WITH NO
|
|
23
|
+
# ERROR AND NO JOB ID; 0.77 MB -> ACCEPTED"} [MEASURED 2026-09-12: capping the long edge at 640
|
|
24
|
+
# makes cosmos3's three views 0.12 MB vs 1.98 MB native]
|
|
25
|
+
MAX_UPLOAD_EDGE = 640
|
|
26
|
+
|
|
27
|
+
# How far the letterboxed content block may sit from where training data sits, in pixels along the
|
|
28
|
+
# padded axis. Deliberately the gateway's number, not a looser local one: a frame accepted here and
|
|
29
|
+
# rejected there (or the reverse) is worse than either rule alone.
|
|
30
|
+
# {backend/sequences/api/contract/geometry.py "_TOLERANCE_PX = 5"}
|
|
31
|
+
ASPECT_TOLERANCE_PX = 5
|
|
32
|
+
|
|
33
|
+
# Wire filter name -> PIL constant. Deliberately small: every entry is a filter some served model's
|
|
34
|
+
# own transform actually uses, so adding one means having read that model's source. `encode` raises
|
|
35
|
+
# on anything not here rather than defaulting -- a silent default is exactly how 0.7.0 shipped
|
|
36
|
+
# LANCZOS against an upstream that uses BILINEAR.
|
|
37
|
+
_FILTERS: dict = {}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def filters() -> dict:
|
|
41
|
+
"""Built on first use, because PIL is an optional dependency and this module imports without it."""
|
|
42
|
+
if not _FILTERS:
|
|
43
|
+
from PIL import Image
|
|
44
|
+
_FILTERS.update(bilinear=Image.BILINEAR, bicubic=Image.BICUBIC,
|
|
45
|
+
# INTER_AREA's closest PIL equivalent; GR00T uses it, though that model
|
|
46
|
+
# declines pre-resizing so this entry is currently unreached.
|
|
47
|
+
area=Image.BOX)
|
|
48
|
+
return _FILTERS
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass(frozen=True)
|
|
52
|
+
class ResizeSpec:
|
|
53
|
+
"""The geometry a model declares, de-coupled from any catalogue type. The SDK builds one from its
|
|
54
|
+
`Model`; the gateway builds one from its `EmbodimentSpec`. Three decisions, made separately:
|
|
55
|
+
|
|
56
|
+
- `client_resize`: the target SIZE, in one of two shapes — `{"size": [w, h]}` for a fixed target,
|
|
57
|
+
or `{"shortest_edge": n}` when the server's step is defined on the short edge. Carries the
|
|
58
|
+
`"filter"` name too. `None` → fall to MAX_UPLOAD_EDGE.
|
|
59
|
+
- `input_resize`: the GEOMETRY — `"pad"` (openpi's resize_with_pad, letterbox) or `"stretch"`
|
|
60
|
+
(unconditional interpolate, cosmos3). `None` treated as `"pad"`.
|
|
61
|
+
- `aspect_ratio`: the trained aspect. A frame outside it (± ASPECT_TOLERANCE_PX on the content
|
|
62
|
+
block) is REFUSED, not reshaped. `None` → no aspect check.
|
|
63
|
+
- `label`: identifies the model in error messages.
|
|
64
|
+
"""
|
|
65
|
+
client_resize: dict | None = None
|
|
66
|
+
input_resize: str | None = None
|
|
67
|
+
aspect_ratio: float | None = None
|
|
68
|
+
label: str = "model"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def refuse_wrong_aspect(img: Any, aspect_ratio: float | None, square: int, label: str) -> None:
|
|
72
|
+
"""
|
|
73
|
+
Raise if `img`'s ratio would letterbox onto a content block the model never trained on.
|
|
74
|
+
|
|
75
|
+
The band is the gateway's, not a new one: the content block may sit within ASPECT_TOLERANCE_PX
|
|
76
|
+
rows of where training data sits, so a frame is never accepted locally and rejected remotely or
|
|
77
|
+
the reverse. 16:9 lands at exactly 224/1.7778 = 126 rows; the band admits 848x480 (126.8) and
|
|
78
|
+
refuses 640x480 (168). {backend/sequences/api/contract/geometry.py `_TOLERANCE_PX = 5`}
|
|
79
|
+
|
|
80
|
+
Advice, not just refusal: a narrower true-to-training crop beats a complete out-of-distribution
|
|
81
|
+
frame — but the caller does the crop, the codec will not do it silently (until 2026-09-16 it
|
|
82
|
+
centre-cropped every frame, making a 4:3 camera "work" while the model ran on a scene missing
|
|
83
|
+
12.5% of its height the caller still believed they had sent).
|
|
84
|
+
"""
|
|
85
|
+
if not aspect_ratio:
|
|
86
|
+
return
|
|
87
|
+
|
|
88
|
+
w, h = img.size
|
|
89
|
+
ar = w / h
|
|
90
|
+
# Where the scene lands inside the square, in pixels along the padded axis.
|
|
91
|
+
extent = square / ar if ar >= 1 else square * ar
|
|
92
|
+
trained = square / aspect_ratio if aspect_ratio >= 1 else square * aspect_ratio
|
|
93
|
+
if abs(extent - trained) <= ASPECT_TOLERANCE_PX:
|
|
94
|
+
return
|
|
95
|
+
|
|
96
|
+
axis = "rows" if ar >= 1 else "columns"
|
|
97
|
+
raise ValueError(
|
|
98
|
+
f"{label} trained on {aspect_ratio:.4f} ({ratio_name(aspect_ratio)}) frames, and this one "
|
|
99
|
+
f"is {w}x{h} ({ratio_name(ar)}). Letterboxed it would fill {extent:.0f} {axis} of "
|
|
100
|
+
f"{square} where training data fills {trained:.0f} — the model would run on a framing it "
|
|
101
|
+
f"has never seen and return plausible but wrong actions. Crop to "
|
|
102
|
+
f"{ratio_name(aspect_ratio)} before calling; do not pad to it. The gateway refuses the same "
|
|
103
|
+
f"frame with HTTP 400, so this is the same rule applied one step earlier."
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def ratio_name(ar: float) -> str:
|
|
108
|
+
"""`1.7778` as `16:9`. Falls back to two decimals for ratios with no common name."""
|
|
109
|
+
for w, h in ((16, 9), (4, 3), (3, 2), (1, 1), (9, 16), (3, 4)):
|
|
110
|
+
if abs(ar - w / h) / (w / h) < 0.01:
|
|
111
|
+
return f"{w}:{h}"
|
|
112
|
+
return f"{ar:.2f}:1"
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def resize_with_pad(img: Any, width: int, height: int, resample: int) -> Any:
|
|
116
|
+
"""
|
|
117
|
+
Scale `img` into `width` x `height` keeping its aspect ratio, padding the remainder black.
|
|
118
|
+
|
|
119
|
+
This is openpi's `resize_with_pad`, line for line, because the server runs that exact function on
|
|
120
|
+
whatever arrives. Doing the same operation here means the server's copy hits its own
|
|
121
|
+
short-circuit (`if images.shape[-3:-1] == (height, width): return images`) and changes nothing,
|
|
122
|
+
so a frame pre-sized here and a frame sent raw reach the model as the same pixels — by
|
|
123
|
+
construction, not by coincidence.
|
|
124
|
+
{openpi packages/openpi-client/src/openpi_client/image_tools.py:44-58 "_RESIZE_WITH_PAD_PIL"}
|
|
125
|
+
|
|
126
|
+
Note the truncation: upstream uses `int()`, not `round()`, on both edges. 320x180 into a 224
|
|
127
|
+
square gives int(180 / (320/224)) = int(126.0) = 126, and reproducing that exactly is the
|
|
128
|
+
difference between agreeing with the server and being one pixel off.
|
|
129
|
+
"""
|
|
130
|
+
from PIL import Image
|
|
131
|
+
|
|
132
|
+
cur_width, cur_height = img.size
|
|
133
|
+
if cur_width == width and cur_height == height:
|
|
134
|
+
return img
|
|
135
|
+
|
|
136
|
+
# The long edge decides the scale, so neither axis can overflow the target.
|
|
137
|
+
ratio = max(cur_width / width, cur_height / height)
|
|
138
|
+
resized_height = int(cur_height / ratio)
|
|
139
|
+
resized_width = int(cur_width / ratio)
|
|
140
|
+
resized = img.resize((resized_width, resized_height), resample=resample)
|
|
141
|
+
|
|
142
|
+
# Black, and centred. `max(0, ...)` guards the case where truncation leaves the scaled edge a
|
|
143
|
+
# pixel longer than the target, which would otherwise paste at a negative offset.
|
|
144
|
+
canvas = Image.new(resized.mode, (width, height), 0)
|
|
145
|
+
canvas.paste(resized, (max(0, int((width - resized_width) / 2)),
|
|
146
|
+
max(0, int((height - resized_height) / 2))))
|
|
147
|
+
return canvas
|
|
File without changes
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""SEQ_* environment variable NAMES — a single source. Values live in ~/.config/sequences/, never here."""
|
|
2
|
+
|
|
3
|
+
DIRECT_SECRET = "SEQ_DIRECT_SECRET" # inbound /act auth on a public pod
|
|
4
|
+
INTERNAL_SECRET = "SEQ_INTERNAL_SECRET" # auth for container → gateway telemetry
|
|
5
|
+
GATEWAY_URL = "SEQ_GATEWAY_URL" # where the container sends telemetry
|
|
6
|
+
VOLUME_ROOT = "SEQ_VOLUME_ROOT" # where fetched weights land
|
|
7
|
+
MANIFEST = "SEQ_MANIFEST" # the policy manifest a container serves
|
|
8
|
+
|
|
9
|
+
# Weight store (Cloudflare R2).
|
|
10
|
+
DIST_ENDPOINT = "SEQ_DIST_ENDPOINT"
|
|
11
|
+
DIST_BUCKET = "SEQ_DIST_BUCKET"
|
|
12
|
+
DIST_KEY = "SEQ_DIST_KEY"
|
|
13
|
+
DIST_SECRET = "SEQ_DIST_SECRET"
|
|
14
|
+
DIST_REGION = "SEQ_DIST_REGION"
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
"""HTTP header names — one source, so sdk / gateway / container never spell them differently."""
|
|
2
|
+
|
|
3
|
+
REQUEST_ID = "X-Seq-Request-Id" # correlation id; == sequence_base.obs.rid.HEADER
|
|
4
|
+
MODEL = "X-Seq-Model" # gateway → worker model routing
|
|
5
|
+
AUTH_SCHEME = "Bearer" # scheme for SEQ_DIRECT_SECRET on the direct-connect path
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""Wire protocol version. Bump on a breaking change to contract / codec / telemetry shapes."""
|
|
2
|
+
|
|
3
|
+
PROTOCOL_VERSION = "1"
|
|
4
|
+
|
|
5
|
+
# The single version stamped into every exported JSON Schema (as `x-contract-version`), so a consumer
|
|
6
|
+
# across deployments / repos can tell which version of a shape it is validating against. MUST change
|
|
7
|
+
# whenever any shared wire shape changes (a field added / removed / retyped) — that is what makes
|
|
8
|
+
# contract evolution mechanically detectable instead of eyeballed. Bump on any shape change.
|
|
9
|
+
CONTRACT_VERSION = "0.4.1" # 0.4.0 -> 0.4.1: 契约2 deploy body completed during S3/G4 — DeploymentPolicy.secrets (name-only passthrough), DeploymentRequest.volumes: list[VolumeAttach{uri,mount,create_if_missing}], concurrency default materialized; telemetry events add load_start stage + emitted_at/rid/dedup_key (O4)
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The shared wire contract — one definition per shape, imported (never redeclared) by the SDK, gateway,
|
|
3
|
+
and container harness. Grouped by the contract number in docs/platform/contracts.md.
|
|
4
|
+
"""
|
|
5
|
+
# contract 4 — /act data plane (obs/action DATA types)
|
|
6
|
+
from sequence_base.contract.act import (
|
|
7
|
+
ActionChunk,
|
|
8
|
+
ActionSpace,
|
|
9
|
+
ActionStep,
|
|
10
|
+
ImageFrame,
|
|
11
|
+
Observation,
|
|
12
|
+
Proprioception,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
# contract 1 — lifecycle flags + policy registry
|
|
16
|
+
from sequence_base.contract.flags import SeqFlags
|
|
17
|
+
from sequence_base.contract.policy import (
|
|
18
|
+
ActionContract,
|
|
19
|
+
Concurrency,
|
|
20
|
+
ImageSpec,
|
|
21
|
+
ImageStep,
|
|
22
|
+
ImageStepApt,
|
|
23
|
+
ImageStepEnv,
|
|
24
|
+
ImageStepRun,
|
|
25
|
+
ImageStepUvPip,
|
|
26
|
+
PolicyRegistration,
|
|
27
|
+
PolicySpec,
|
|
28
|
+
VolumeMount,
|
|
29
|
+
VolumeRef,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
# contract 2 — deployments
|
|
33
|
+
from sequence_base.contract.deployments import (
|
|
34
|
+
MAX_CODE_BUNDLE_BYTES,
|
|
35
|
+
Autoscaler,
|
|
36
|
+
CodeBundle,
|
|
37
|
+
DeploymentAccepted,
|
|
38
|
+
DeploymentPolicy,
|
|
39
|
+
DeploymentRequest,
|
|
40
|
+
DeploymentState,
|
|
41
|
+
DeploymentStatusResponse,
|
|
42
|
+
Resources,
|
|
43
|
+
Timeouts,
|
|
44
|
+
VolumeAttach,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
# contract 3 — connect / lease
|
|
48
|
+
from sequence_base.contract.connect import ConnectRequest, ConnectResponse, Lease
|
|
49
|
+
|
|
50
|
+
# contract 5 — benchmark registration + eval run / report / adapter
|
|
51
|
+
from sequence_base.contract.benchmarks import (
|
|
52
|
+
BenchmarkRegisterRequest,
|
|
53
|
+
BenchmarkSpec,
|
|
54
|
+
Conditions,
|
|
55
|
+
RenderSpec,
|
|
56
|
+
Step,
|
|
57
|
+
Variant,
|
|
58
|
+
)
|
|
59
|
+
from sequence_base.contract.eval import (
|
|
60
|
+
AdapterKind,
|
|
61
|
+
AdapterTransform,
|
|
62
|
+
ClampInfo,
|
|
63
|
+
Estimate,
|
|
64
|
+
EvalDryRunResponse,
|
|
65
|
+
EvalReport,
|
|
66
|
+
EvalRequest,
|
|
67
|
+
EvalResponse,
|
|
68
|
+
EvalStatus,
|
|
69
|
+
LossyTransform,
|
|
70
|
+
MetricAgg,
|
|
71
|
+
RolloutResult,
|
|
72
|
+
SuiteAgg,
|
|
73
|
+
TaskAgg,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
# contract 6 — accounts / secret / billing
|
|
77
|
+
from sequence_base.contract.accounts import (
|
|
78
|
+
BootstrapResponse,
|
|
79
|
+
SecretCreateRequest,
|
|
80
|
+
SecretInfo,
|
|
81
|
+
SecretRef,
|
|
82
|
+
UsageReport,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
# shared error bodies
|
|
86
|
+
from sequence_base.contract.errors import ErrorBody, ErrorCode, WarmingError
|
|
87
|
+
|
|
88
|
+
__all__ = [
|
|
89
|
+
# contract 4
|
|
90
|
+
"ImageFrame", "Proprioception", "Observation", "ActionSpace", "ActionStep", "ActionChunk",
|
|
91
|
+
# contract 1
|
|
92
|
+
"SeqFlags",
|
|
93
|
+
"ImageStepUvPip", "ImageStepApt", "ImageStepEnv", "ImageStepRun", "ImageStep", "ImageSpec",
|
|
94
|
+
"VolumeMount", "VolumeRef", "Concurrency", "ActionContract",
|
|
95
|
+
"PolicySpec", "PolicyRegistration",
|
|
96
|
+
# contract 2
|
|
97
|
+
"MAX_CODE_BUNDLE_BYTES", "CodeBundle",
|
|
98
|
+
"Resources", "Autoscaler", "Timeouts", "VolumeAttach", "DeploymentPolicy", "DeploymentRequest",
|
|
99
|
+
"DeploymentAccepted", "DeploymentState", "DeploymentStatusResponse",
|
|
100
|
+
# contract 3
|
|
101
|
+
"Lease", "ConnectRequest", "ConnectResponse",
|
|
102
|
+
# contract 5
|
|
103
|
+
"Variant", "Conditions", "RenderSpec", "Step", "BenchmarkSpec", "BenchmarkRegisterRequest",
|
|
104
|
+
"EvalRequest", "ClampInfo", "Estimate", "EvalResponse", "EvalDryRunResponse", "RolloutResult",
|
|
105
|
+
"EvalStatus", "SuiteAgg", "TaskAgg", "MetricAgg", "LossyTransform", "EvalReport",
|
|
106
|
+
"AdapterKind", "AdapterTransform",
|
|
107
|
+
# contract 6
|
|
108
|
+
"SecretRef", "SecretCreateRequest", "SecretInfo", "BootstrapResponse", "UsageReport",
|
|
109
|
+
# errors
|
|
110
|
+
"ErrorCode", "ErrorBody", "WarmingError",
|
|
111
|
+
]
|