sequence-base 0.4.1__tar.gz → 0.4.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sequence_base-0.4.1 → sequence_base-0.4.3}/.gitignore +1 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/PKG-INFO +2 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/pyproject.toml +4 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/__init__.py +4 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/codec/__init__.py +6 -2
- sequence_base-0.4.3/sequence_base/codec/image.py +210 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/constants/version.py +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/contract/__init__.py +5 -2
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/contract/io.py +5 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/contract/policy.py +30 -2
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/presets/robot_arm.py +68 -2
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schema_export.py +4 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ActRequest.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ActResponse.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ActionChunk.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ActionContract.json +17 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ActionStep.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/AdapterTransform.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/AddLocal.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/Autoscaler.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/BenchmarkConditionSpec.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/BenchmarkRegisterRequest.json +59 -2
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/BenchmarkSpec.json +44 -3
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/BenchmarkStatusResponse.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/BootstrapResponse.json +1 -1
- sequence_base-0.4.3/sequence_base/schemas/Camera.json +76 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ClampInfo.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/CodeBundle.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/Concurrency.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/Conditions.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ConnectRequest.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ConnectResponse.json +56 -2
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/DeploymentAccepted.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/DeploymentPolicy.json +56 -2
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/DeploymentRequest.json +98 -3
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/DeploymentStatusResponse.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/Embodiment.json +56 -2
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ErrorBody.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ErrorCodes.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/Estimate.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/EvalDryRunResponse.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/EvalReport.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/EvalRequest.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/EvalResponse.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ImageFrame.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ImageSpec.json +43 -2
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ImageStepAddLocal.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ImageStepApt.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ImageStepEnv.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ImageStepRun.json +1 -1
- sequence_base-0.4.3/sequence_base/schemas/ImageStepTemplateFiles.json +41 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ImageStepUvPip.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/InferParam.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/Lease.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/LossyTransform.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/MetricAgg.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/Observation.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/ObservationContract.json +40 -2
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/PolicyRegistration.json +99 -4
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/PolicySpec.json +98 -3
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/Proprioception.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/RenderSpec.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/Resources.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/RolloutResult.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/SecretCreateRequest.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/SecretInfo.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/SecretRef.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/SeqFlags.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/Step.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/SuiteAgg.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/TaskAgg.json +1 -1
- sequence_base-0.4.3/sequence_base/schemas/TemplateFiles.json +26 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/Timeouts.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/UploadTarget.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/Variant.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/VolumeAttach.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/VolumeFile.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/VolumeInfo.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/VolumeMount.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/VolumeRef.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/VolumeUploadRequest.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/VolumeUploadResponse.json +1 -1
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/schemas/WarmingError.json +1 -1
- sequence_base-0.4.1/sequence_base/codec/image.py +0 -119
- sequence_base-0.4.1/sequence_base/schemas/Camera.json +0 -38
- {sequence_base-0.4.1 → sequence_base-0.4.3}/README.md +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/auth.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/constants/__init__.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/constants/headers.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/contract/accounts.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/contract/act.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/contract/benchmarks.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/contract/connect.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/contract/deployments.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/contract/errors.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/contract/eval.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/contract/flags.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/contract/templates.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/contract/volumes.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/contract/wire.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/errors/__init__.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/errors/transport.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/obs/__init__.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/obs/log.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/obs/rid.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/presets/__init__.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/telemetry/__init__.py +0 -0
- {sequence_base-0.4.1 → sequence_base-0.4.3}/sequence_base/telemetry/events.py +0 -0
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sequence-base
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.3
|
|
4
4
|
Summary: Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness.
|
|
5
|
+
Project-URL: Changelog, https://generalsequences.com/docs/changelog/
|
|
5
6
|
Requires-Python: >=3.10
|
|
6
7
|
Requires-Dist: pydantic>=2.0
|
|
7
8
|
Provides-Extra: codec
|
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sequence-base"
|
|
3
|
-
version = "0.4.
|
|
3
|
+
version = "0.4.3"
|
|
4
4
|
description = "Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness."
|
|
5
5
|
requires-python = ">=3.10"
|
|
6
6
|
dependencies = ["pydantic>=2.0"]
|
|
7
7
|
|
|
8
|
+
[project.urls]
|
|
9
|
+
Changelog = "https://generalsequences.com/docs/changelog/"
|
|
10
|
+
|
|
8
11
|
[project.optional-dependencies]
|
|
9
12
|
codec = ["pillow>=10", "numpy>=1.26"]
|
|
10
13
|
test = ["pytest", "jsonschema>=4"]
|
|
@@ -18,6 +18,9 @@ from sequence_base.contract import (
|
|
|
18
18
|
ActResponse,
|
|
19
19
|
AddLocal,
|
|
20
20
|
ImageStepAddLocal,
|
|
21
|
+
FROZEN_PATH,
|
|
22
|
+
TemplateFiles,
|
|
23
|
+
ImageStepTemplateFiles,
|
|
21
24
|
Camera,
|
|
22
25
|
Embodiment,
|
|
23
26
|
ObservationContract,
|
|
@@ -126,7 +129,7 @@ __all__ = [
|
|
|
126
129
|
"ActRequest", "ActResponse",
|
|
127
130
|
|
|
128
131
|
"SeqFlags", "ImageStepUvPip", "ImageStepApt", "ImageStepEnv", "ImageStepRun", "AddLocal", "ImageStepAddLocal",
|
|
129
|
-
"ImageStep", "ImageSpec",
|
|
132
|
+
"FROZEN_PATH", "TemplateFiles", "ImageStepTemplateFiles", "ImageStep", "ImageSpec",
|
|
130
133
|
"VolumeMount", "VolumeRef", "Concurrency", "ActionContract", "Camera", "ObservationContract", "Embodiment",
|
|
131
134
|
"PolicySpec", "PolicyRegistration",
|
|
132
135
|
|
|
@@ -9,11 +9,15 @@ decodes, installs the light core and never imports this.
|
|
|
9
9
|
"""
|
|
10
10
|
from sequence_base.codec.image import (
|
|
11
11
|
DEFAULT_JPEG_QUALITY,
|
|
12
|
-
|
|
12
|
+
FILTERS,
|
|
13
|
+
MODES,
|
|
13
14
|
decode,
|
|
14
15
|
decode_depth,
|
|
15
16
|
encode,
|
|
16
17
|
encode_depth,
|
|
18
|
+
image_size,
|
|
19
|
+
resize_image,
|
|
17
20
|
)
|
|
18
21
|
|
|
19
|
-
__all__ = ["encode", "decode", "encode_depth", "decode_depth", "
|
|
22
|
+
__all__ = ["encode", "decode", "encode_depth", "decode_depth", "resize_image", "image_size", "DEFAULT_JPEG_QUALITY",
|
|
23
|
+
"MODES", "FILTERS"]
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The image codec: one camera frame <-> the base64 string the wire carries.
|
|
3
|
+
|
|
4
|
+
`encode` (frame -> base64 JPEG) is the SDK's client-side path; `decode` (base64 -> uint8 RGB array)
|
|
5
|
+
is the container/worker's server-side path. They live together so the wire format — JPEG, quality
|
|
6
|
+
95, RGB — is defined once and provable by a round-trip test, and so the SDK and the worker cannot
|
|
7
|
+
drift on what they put on and take off the wire.
|
|
8
|
+
|
|
9
|
+
Pillow is imported lazily inside each function, never at module scope: a caller who only passes
|
|
10
|
+
pre-encoded strings must not need it installed.
|
|
11
|
+
|
|
12
|
+
Nothing here fits a frame to a policy behind the caller's back (contract 0.5.2): `encode` sends the frame it is given,
|
|
13
|
+
`resize_image` is the tool the robot calls to fit one the way a camera's contract says the policy was trained, and
|
|
14
|
+
`image_size` reads a frame's width and height from its header - what the worker checks a declared camera against.
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import base64
|
|
21
|
+
import binascii
|
|
22
|
+
import io
|
|
23
|
+
import struct
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
DEFAULT_JPEG_QUALITY = 95
|
|
34
|
+
|
|
35
|
+
def encode(frame: Any, *, quality: int = DEFAULT_JPEG_QUALITY) -> str:
|
|
36
|
+
"""
|
|
37
|
+
One camera frame to the base64 image that `ImageFrame.data` carries.
|
|
38
|
+
|
|
39
|
+
`frame` may be a numpy array, a PIL image, raw `bytes` already encoded, or a `str` already base64
|
|
40
|
+
— the last two pass through untouched, so a caller with their own pipeline is not forced through
|
|
41
|
+
this one. Codec is JPEG at `quality`, unconditionally, at the frame's own size: fitting it to a
|
|
42
|
+
policy is `resize_image`, called by the robot. (Before contract 0.5.2 a frame over 640 px on its
|
|
43
|
+
long edge was scaled down to it here.)
|
|
44
|
+
"""
|
|
45
|
+
if isinstance(frame, str):
|
|
46
|
+
return frame
|
|
47
|
+
if isinstance(frame, (bytes, bytearray)):
|
|
48
|
+
return base64.b64encode(bytes(frame)).decode()
|
|
49
|
+
|
|
50
|
+
try:
|
|
51
|
+
from PIL import Image
|
|
52
|
+
except ImportError as e:
|
|
53
|
+
raise ImportError(
|
|
54
|
+
"encoding an array or PIL image needs Pillow: pip install 'sequence-base[codec]'. "
|
|
55
|
+
"Pass an already-encoded base64 str or bytes to avoid it."
|
|
56
|
+
) from e
|
|
57
|
+
|
|
58
|
+
img = frame if hasattr(frame, "save") else Image.fromarray(frame)
|
|
59
|
+
if img.mode != "RGB":
|
|
60
|
+
img = img.convert("RGB")
|
|
61
|
+
buf = io.BytesIO()
|
|
62
|
+
img.save(buf, format="JPEG", quality=quality)
|
|
63
|
+
return base64.b64encode(buf.getvalue()).decode()
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def decode(data: str) -> "Any":
|
|
67
|
+
"""
|
|
68
|
+
base64 JPEG/PNG -> HxWx3 uint8 RGB array — the worker's server-side inverse of `encode`.
|
|
69
|
+
|
|
70
|
+
This is what turns `ImageFrame.data` back into pixels for the model. Send JPEG q95: 16x smaller
|
|
71
|
+
than a JSON int array, ~0.5% action delta. {sequences-models worker handler decode_image}
|
|
72
|
+
|
|
73
|
+
Returns a numpy array; numpy is part of the `[codec]` extra alongside Pillow.
|
|
74
|
+
"""
|
|
75
|
+
import numpy as np
|
|
76
|
+
from PIL import Image
|
|
77
|
+
|
|
78
|
+
raw = base64.b64decode(data, validate=False)
|
|
79
|
+
return np.asarray(Image.open(io.BytesIO(raw)).convert("RGB"), dtype=np.uint8)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
MODES = ("pad", "stretch")
|
|
87
|
+
FILTERS = ("nearest", "box", "bilinear", "hamming", "bicubic", "lanczos")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _to_image(frame: Any) -> "Any":
|
|
91
|
+
"""A frame as an RGB PIL image: a PIL image, encoded bytes (JPEG / PNG), base64 text (a data: URI too), or an
|
|
92
|
+
HxWx3 uint8 array (anything Image.fromarray takes)."""
|
|
93
|
+
from PIL import Image
|
|
94
|
+
|
|
95
|
+
if isinstance(frame, str):
|
|
96
|
+
frame = base64.b64decode(frame.split(",", 1)[1] if frame.startswith("data:") else frame, validate=False)
|
|
97
|
+
if isinstance(frame, (bytes, bytearray)):
|
|
98
|
+
img = Image.open(io.BytesIO(bytes(frame)))
|
|
99
|
+
elif hasattr(frame, "save"):
|
|
100
|
+
img = frame
|
|
101
|
+
else:
|
|
102
|
+
img = Image.fromarray(frame)
|
|
103
|
+
return img if img.mode == "RGB" else img.convert("RGB")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def resize_image(frame: Any, width: int, height: int, *, mode: str, filter: str) -> "Any":
|
|
107
|
+
"""
|
|
108
|
+
One camera frame fitted to `width` x `height`, as the robot sends it to a camera that reads that size: an RGB PIL
|
|
109
|
+
image, for `encode` next.
|
|
110
|
+
|
|
111
|
+
`mode` "pad" scales it into the box keeping its aspect ratio and pads the rest black, centred - openpi's
|
|
112
|
+
resize_with_pad exactly (both edges truncated with int()), what the templates' client.py ran; "stretch" scales it
|
|
113
|
+
to the box. `filter` is the resampling filter, one of FILTERS. Neither has a default: they are how the policy was
|
|
114
|
+
trained, and a policy's contract names them (Camera.resize, Camera.filter). A frame already that size is returned
|
|
115
|
+
as it is. `frame` is a PIL image, encoded JPEG / PNG bytes, base64 text, or an HxWx3 uint8 array.
|
|
116
|
+
"""
|
|
117
|
+
if mode not in MODES:
|
|
118
|
+
raise ValueError(f"mode must be one of {MODES}, got {mode!r}")
|
|
119
|
+
if filter not in FILTERS:
|
|
120
|
+
raise ValueError(f"filter must be one of {FILTERS}, got {filter!r}")
|
|
121
|
+
if int(width) <= 0 or int(height) <= 0:
|
|
122
|
+
raise ValueError(f"the size must be positive, got {width}x{height}")
|
|
123
|
+
from PIL import Image
|
|
124
|
+
|
|
125
|
+
img = _to_image(frame)
|
|
126
|
+
if img.size == (width, height):
|
|
127
|
+
return img
|
|
128
|
+
resample = Image.Resampling[filter.upper()]
|
|
129
|
+
if mode == "stretch":
|
|
130
|
+
return img.resize((width, height), resample=resample)
|
|
131
|
+
cur_width, cur_height = img.size
|
|
132
|
+
ratio = max(cur_width / width, cur_height / height)
|
|
133
|
+
resized_width, resized_height = int(cur_width / ratio), int(cur_height / ratio)
|
|
134
|
+
resized = img.resize((resized_width, resized_height), resample=resample)
|
|
135
|
+
canvas = Image.new(resized.mode, (width, height), 0)
|
|
136
|
+
canvas.paste(resized, (max(0, int((width - resized_width) / 2)), max(0, int((height - resized_height) / 2))))
|
|
137
|
+
return canvas
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def image_size(data: str) -> tuple[int, int]:
|
|
141
|
+
"""
|
|
142
|
+
(width, height) of the frame `ImageFrame.data` carries - base64 JPEG or PNG, a data: URI's prefix taken off -
|
|
143
|
+
read from its header alone. ValueError for anything else.
|
|
144
|
+
|
|
145
|
+
No Pillow: the SDK checks a frame with it before sending, where Pillow may not be installed, and Pillow refuses a
|
|
146
|
+
header that declares a huge size (DecompressionBombError), which is then not "not an image" at all. A PNG's size
|
|
147
|
+
is in its IHDR; a JPEG's in the first start-of-frame segment (SOF0-SOF15, but not C4 / C8 / CC, which are not
|
|
148
|
+
frames).
|
|
149
|
+
"""
|
|
150
|
+
text = data.split(",", 1)[1] if isinstance(data, str) and data.startswith("data:") else data
|
|
151
|
+
try:
|
|
152
|
+
raw = base64.b64decode(text, validate=False)
|
|
153
|
+
except (binascii.Error, ValueError, TypeError) as e:
|
|
154
|
+
raise ValueError("not base64") from e
|
|
155
|
+
if raw[:8] == b"\x89PNG\r\n\x1a\n" and raw[12:16] == b"IHDR" and len(raw) >= 24:
|
|
156
|
+
return struct.unpack(">II", raw[16:24])
|
|
157
|
+
if raw[:2] != b"\xff\xd8":
|
|
158
|
+
raise ValueError("not a JPEG or PNG image")
|
|
159
|
+
i = 2
|
|
160
|
+
while i + 4 <= len(raw):
|
|
161
|
+
if raw[i] != 0xFF:
|
|
162
|
+
raise ValueError("a JPEG whose segments do not line up")
|
|
163
|
+
marker = raw[i + 1]
|
|
164
|
+
if marker == 0xFF:
|
|
165
|
+
i += 1
|
|
166
|
+
continue
|
|
167
|
+
if marker == 0x01 or 0xD0 <= marker <= 0xD8:
|
|
168
|
+
i += 2
|
|
169
|
+
continue
|
|
170
|
+
if marker in (0xD9, 0xDA):
|
|
171
|
+
break
|
|
172
|
+
if 0xC0 <= marker <= 0xCF and marker not in (0xC4, 0xC8, 0xCC) and i + 9 <= len(raw):
|
|
173
|
+
height, width = struct.unpack(">HH", raw[i + 5:i + 9])
|
|
174
|
+
return width, height
|
|
175
|
+
i += 2 + int.from_bytes(raw[i + 2:i + 4], "big")
|
|
176
|
+
raise ValueError("a JPEG with no frame header")
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
_DEPTH_UNITS = {"m": 1000.0, "mm": 1.0}
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def encode_depth(depth: Any, *, unit: str = "m") -> str:
|
|
184
|
+
"""A depth map -> the base64 16-bit PNG a depth frame (ImageFrame kind="depth") carries: millimetres as uint16, 0
|
|
185
|
+
where there is no reading (NaN, inf, <= 0), clipped at 65.535 m. `unit` is what the array holds: "m" (float metres,
|
|
186
|
+
the default) or "mm". Lossless, unlike the JPEG colour frames take — a depth error is a geometry error.
|
|
187
|
+
|
|
188
|
+
"""
|
|
189
|
+
import numpy as np
|
|
190
|
+
from PIL import Image
|
|
191
|
+
|
|
192
|
+
if unit not in _DEPTH_UNITS:
|
|
193
|
+
raise ValueError(f"unit must be one of {sorted(_DEPTH_UNITS)}, got {unit!r}")
|
|
194
|
+
arr = np.asarray(depth, dtype=np.float64)
|
|
195
|
+
if arr.ndim != 2:
|
|
196
|
+
raise ValueError(f"a depth map is HxW, got shape {arr.shape}")
|
|
197
|
+
mm = arr * _DEPTH_UNITS[unit]
|
|
198
|
+
mm = np.where(np.isfinite(mm) & (mm > 0), mm, 0.0)
|
|
199
|
+
buf = io.BytesIO()
|
|
200
|
+
Image.fromarray(np.clip(np.rint(mm), 0, 65535).astype(np.uint16)).save(buf, format="PNG")
|
|
201
|
+
return base64.b64encode(buf.getvalue()).decode()
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def decode_depth(data: str) -> "Any":
|
|
205
|
+
"""A depth frame's data -> HxW uint16 millimetres (0: no reading) — the worker's inverse of `encode_depth`."""
|
|
206
|
+
import numpy as np
|
|
207
|
+
from PIL import Image
|
|
208
|
+
|
|
209
|
+
raw = base64.b64decode(data, validate=False)
|
|
210
|
+
return np.asarray(Image.open(io.BytesIO(raw)), dtype=np.uint16)
|
|
@@ -31,6 +31,9 @@ from sequence_base.contract.policy import (
|
|
|
31
31
|
ImageStepUvPip,
|
|
32
32
|
AddLocal,
|
|
33
33
|
ImageStepAddLocal,
|
|
34
|
+
FROZEN_PATH,
|
|
35
|
+
TemplateFiles,
|
|
36
|
+
ImageStepTemplateFiles,
|
|
34
37
|
PolicyRegistration,
|
|
35
38
|
PolicySpec,
|
|
36
39
|
VolumeMount,
|
|
@@ -117,8 +120,8 @@ __all__ = [
|
|
|
117
120
|
"ActRequest", "ActResponse",
|
|
118
121
|
|
|
119
122
|
"SeqFlags",
|
|
120
|
-
"ImageStepUvPip", "ImageStepApt", "ImageStepEnv", "ImageStepRun", "AddLocal", "ImageStepAddLocal",
|
|
121
|
-
"ImageSpec",
|
|
123
|
+
"ImageStepUvPip", "ImageStepApt", "ImageStepEnv", "ImageStepRun", "AddLocal", "ImageStepAddLocal",
|
|
124
|
+
"TemplateFiles", "ImageStepTemplateFiles", "FROZEN_PATH", "ImageStep", "ImageSpec",
|
|
122
125
|
"VolumeMount", "VolumeRef", "Concurrency", "ActionContract", "Camera", "ObservationContract", "Embodiment",
|
|
123
126
|
"PolicySpec", "PolicyRegistration",
|
|
124
127
|
|
|
@@ -7,11 +7,15 @@ Declaring one is optional since contract 0.4.14: a deployment that declares none
|
|
|
7
7
|
handed to the robot at connect.
|
|
8
8
|
"""
|
|
9
9
|
from sequence_base.presets.robot_arm import (
|
|
10
|
+
PAYLOAD_HINT,
|
|
10
11
|
PROPRIO_FIELDS,
|
|
11
12
|
ActionContract,
|
|
12
13
|
Camera,
|
|
13
14
|
Embodiment,
|
|
14
15
|
ObservationContract,
|
|
16
|
+
fit_hint,
|
|
17
|
+
frame_size_problem,
|
|
15
18
|
)
|
|
16
19
|
|
|
17
|
-
__all__ = ["ActionContract", "PROPRIO_FIELDS", "Camera", "ObservationContract", "Embodiment"
|
|
20
|
+
__all__ = ["ActionContract", "PROPRIO_FIELDS", "Camera", "ObservationContract", "Embodiment", "frame_size_problem",
|
|
21
|
+
"fit_hint", "PAYLOAD_HINT"]
|
|
@@ -32,6 +32,9 @@ __all__ = [
|
|
|
32
32
|
"ImageStepRun",
|
|
33
33
|
"AddLocal",
|
|
34
34
|
"ImageStepAddLocal",
|
|
35
|
+
"FROZEN_PATH",
|
|
36
|
+
"TemplateFiles",
|
|
37
|
+
"ImageStepTemplateFiles",
|
|
35
38
|
"ImageStep",
|
|
36
39
|
"ImageSpec",
|
|
37
40
|
"VolumeMount",
|
|
@@ -101,14 +104,36 @@ class ImageStepAddLocal(BaseModel):
|
|
|
101
104
|
add_local: AddLocal
|
|
102
105
|
|
|
103
106
|
|
|
104
|
-
|
|
107
|
+
|
|
108
|
+
FROZEN_PATH = "/seq/frozen"
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class TemplateFiles(BaseModel):
|
|
112
|
+
"""An official template version's frozen build files - the source at its pinned commit, every Python package and
|
|
113
|
+
system package its image installs, the assets it downloads - which the platform keeps itself, so a build never
|
|
114
|
+
reaches the sites they came from (`seq.Image.template_files`). Each run_commands step after this one reads them,
|
|
115
|
+
read-only, at FROZEN_PATH; nothing of them stays in the image unless a step copies or installs it.
|
|
116
|
+
"""
|
|
117
|
+
|
|
118
|
+
model_config = ConfigDict(extra="forbid")
|
|
119
|
+
|
|
120
|
+
template: str = Field(..., pattern=r"^[a-z0-9][a-z0-9-]{0,63}$", description="the template's name, as seq init takes it")
|
|
121
|
+
version: int = Field(..., ge=1, le=999_999_999, description="the template version whose files these are")
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class ImageStepTemplateFiles(BaseModel):
|
|
125
|
+
model_config = ConfigDict(extra="forbid")
|
|
126
|
+
template_files: TemplateFiles
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
ImageStep = ImageStepUvPip | ImageStepApt | ImageStepEnv | ImageStepRun | ImageStepAddLocal | ImageStepTemplateFiles
|
|
105
130
|
|
|
106
131
|
|
|
107
132
|
class ImageSpec(BaseModel):
|
|
108
133
|
"""Declarative image, built by the platform. Exactly one of
|
|
109
134
|
`base` (a pullable registry ref) or `dockerfile` (inline Dockerfile source) is set — the Dockerfile
|
|
110
135
|
is its OWN component, never smuggled into the `base` string, so the gateway can tell a
|
|
111
|
-
registry build from a Dockerfile build. `steps` is an ordered list of the
|
|
136
|
+
registry build from a Dockerfile build. `steps` is an ordered list of the step variants above
|
|
112
137
|
(a bare list[dict] is refused), extending whichever base was chosen."""
|
|
113
138
|
|
|
114
139
|
model_config = ConfigDict(extra="forbid")
|
|
@@ -130,6 +155,9 @@ class ImageSpec(BaseModel):
|
|
|
130
155
|
"(inline Dockerfile source); got "
|
|
131
156
|
+ ("both" if self.base is not None else "neither")
|
|
132
157
|
)
|
|
158
|
+
|
|
159
|
+
if sum(isinstance(s, ImageStepTemplateFiles) for s in self.steps) > 1:
|
|
160
|
+
raise ValueError("an image takes one template_files step - the files of one template version")
|
|
133
161
|
return self
|
|
134
162
|
|
|
135
163
|
|
|
@@ -12,6 +12,7 @@ resolving to these same classes.
|
|
|
12
12
|
"""
|
|
13
13
|
from __future__ import annotations
|
|
14
14
|
|
|
15
|
+
import math
|
|
15
16
|
from enum import Enum
|
|
16
17
|
from typing import Any, Literal
|
|
17
18
|
|
|
@@ -126,8 +127,19 @@ class ActionContract(BaseModel):
|
|
|
126
127
|
|
|
127
128
|
on_violation: Literal["clip", "refuse"] | None = None
|
|
128
129
|
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
binarize: dict[int, float] | None = Field(
|
|
134
|
+
default=None, description="dimension -> threshold: the client gives the robot 1.0 above it, 0.0 otherwise")
|
|
135
|
+
|
|
129
136
|
@model_validator(mode="after")
|
|
130
137
|
def _limits_fit(self) -> "ActionContract":
|
|
138
|
+
for d, threshold in (self.binarize or {}).items():
|
|
139
|
+
if not 0 <= d < self.action_dim:
|
|
140
|
+
raise ValueError(f"binarize names dimension {d}; action_dim is {self.action_dim}")
|
|
141
|
+
if not math.isfinite(threshold):
|
|
142
|
+
raise ValueError(f"binarize[{d}] is {threshold}, not a finite threshold")
|
|
131
143
|
if self.bounds is not None:
|
|
132
144
|
if len(self.bounds) != self.action_dim:
|
|
133
145
|
raise ValueError(f"bounds has {len(self.bounds)} [low, high] pairs; action_dim is {self.action_dim}")
|
|
@@ -148,8 +160,10 @@ PROPRIO_FIELDS = ("joint_positions", "joint_velocities", "end_effector_pose", "g
|
|
|
148
160
|
|
|
149
161
|
|
|
150
162
|
class Camera(BaseModel):
|
|
151
|
-
"""One camera view a policy reads: the size
|
|
152
|
-
|
|
163
|
+
"""One camera view a policy reads: the size its colour frames arrive at. The worker refuses a frame of another size
|
|
164
|
+
(400 invalid_request naming the view, the size it got and the SDK call that fits it) and never resizes one: the
|
|
165
|
+
robot fits its frames before sending - with `sequence_ai.resize_image`, or `policy.prepare_image`, which fits a frame
|
|
166
|
+
the way `resize` and `filter` say the policy was trained."""
|
|
153
167
|
|
|
154
168
|
model_config = ConfigDict(extra="forbid")
|
|
155
169
|
|
|
@@ -160,6 +174,58 @@ class Camera(BaseModel):
|
|
|
160
174
|
depth: bool | None = Field(default=None, description="the view also sends a depth frame (ImageFrame kind=depth)")
|
|
161
175
|
|
|
162
176
|
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
resize: Literal["pad", "stretch"] | None = Field(
|
|
182
|
+
default=None, description='how a frame is fitted to width x height: "pad" (letterbox) or "stretch"')
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
filter: Literal["nearest", "box", "bilinear", "hamming", "bicubic", "lanczos"] | None = Field(
|
|
187
|
+
default=None, description="the resampling filter a frame is fitted with (Pillow's names)")
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def frame_size_problem(view: str, index: int, got: tuple[int, int] | None, camera: dict[str, Any]) -> str | None:
|
|
196
|
+
"""What is wrong with frame `index` of camera `view` (counted in the order the request carries that view's frames,
|
|
197
|
+
from 0), read as `got` (width, height; None: not a JPEG or PNG image), against the camera the contract declares;
|
|
198
|
+
None when it fits."""
|
|
199
|
+
if got is None:
|
|
200
|
+
return f"camera {view!r} frame {index} is not a JPEG or PNG image"
|
|
201
|
+
want = (int(camera["width"]), int(camera["height"]))
|
|
202
|
+
if tuple(got) == want:
|
|
203
|
+
return None
|
|
204
|
+
return f"camera {view!r} frame {index} is {got[0]}x{got[1]}; the policy reads {want[0]}x{want[1]}"
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def fit_hint(view: str, camera: dict[str, Any], *, more: bool = False) -> str:
|
|
208
|
+
"""The next step for a frame of camera `view` that does not fit: the SDK call that fits it as the contract says
|
|
209
|
+
the policy was trained, or, when the contract does not say, the call with the two choices left to the caller -
|
|
210
|
+
never a guess. `more`: other views did not fit either (details.sizes lists them)."""
|
|
211
|
+
width, height = int(camera["width"]), int(camera["height"])
|
|
212
|
+
mode, flt = camera.get("resize"), camera.get("filter")
|
|
213
|
+
if mode and flt:
|
|
214
|
+
hint = (f'fit each frame on the robot before sending: policy.prepare_image("{view}", frame), or '
|
|
215
|
+
f'sequence_ai.resize_image(frame, {width}, {height}, mode="{mode}", filter="{flt}")')
|
|
216
|
+
else:
|
|
217
|
+
hint = (f"fit each frame on the robot before sending: sequence_ai.resize_image(frame, {width}, {height}, "
|
|
218
|
+
f"mode=..., filter=...) with the mode and filter the policy was trained with; this deployment does "
|
|
219
|
+
f"not declare them")
|
|
220
|
+
return hint + (" (the same for each view in details.sizes)" if more else "")
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
PAYLOAD_HINT = ("fit each camera frame on the robot before sending (sequence_ai.resize_image, or policy.prepare_image "
|
|
226
|
+
"for a deployment that declares how): a frame at the size a policy reads is a few kB")
|
|
227
|
+
|
|
228
|
+
|
|
163
229
|
class ObservationContract(BaseModel):
|
|
164
230
|
"""What a policy reads, declared once: the worker checks every /act against it before inference, the
|
|
165
231
|
client assembles to it. An observation that does not match is refused with what is missing — it is never
|
|
@@ -92,6 +92,8 @@ from sequence_base.contract.policy import (
|
|
|
92
92
|
AddLocal,
|
|
93
93
|
ImageStepAddLocal,
|
|
94
94
|
ImageStepRun,
|
|
95
|
+
ImageStepTemplateFiles,
|
|
96
|
+
TemplateFiles,
|
|
95
97
|
ImageStepUvPip,
|
|
96
98
|
PolicyRegistration,
|
|
97
99
|
PolicySpec,
|
|
@@ -120,6 +122,8 @@ WIRE_MODELS: dict[str, type[BaseModel]] = {
|
|
|
120
122
|
"ImageStepRun": ImageStepRun,
|
|
121
123
|
"AddLocal": AddLocal,
|
|
122
124
|
"ImageStepAddLocal": ImageStepAddLocal,
|
|
125
|
+
"TemplateFiles": TemplateFiles,
|
|
126
|
+
"ImageStepTemplateFiles": ImageStepTemplateFiles,
|
|
123
127
|
"ImageSpec": ImageSpec,
|
|
124
128
|
"VolumeMount": VolumeMount,
|
|
125
129
|
"VolumeRef": VolumeRef,
|
|
@@ -23,6 +23,22 @@
|
|
|
23
23
|
"action_space": {
|
|
24
24
|
"$ref": "#/$defs/ActionSpace"
|
|
25
25
|
},
|
|
26
|
+
"binarize": {
|
|
27
|
+
"anyOf": [
|
|
28
|
+
{
|
|
29
|
+
"additionalProperties": {
|
|
30
|
+
"type": "number"
|
|
31
|
+
},
|
|
32
|
+
"type": "object"
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"type": "null"
|
|
36
|
+
}
|
|
37
|
+
],
|
|
38
|
+
"default": null,
|
|
39
|
+
"description": "dimension -> threshold: the client gives the robot 1.0 above it, 0.0 otherwise",
|
|
40
|
+
"title": "Binarize"
|
|
41
|
+
},
|
|
26
42
|
"bounds": {
|
|
27
43
|
"anyOf": [
|
|
28
44
|
{
|
|
@@ -100,5 +116,5 @@
|
|
|
100
116
|
],
|
|
101
117
|
"title": "ActionContract",
|
|
102
118
|
"type": "object",
|
|
103
|
-
"x-contract-version": "0.5.
|
|
119
|
+
"x-contract-version": "0.5.3"
|
|
104
120
|
}
|