sequence-base 0.4.30__tar.gz → 0.4.31__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sequence_base-0.4.30 → sequence_base-0.4.31}/PKG-INFO +5 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/pyproject.toml +2 -1
- sequence_base-0.4.31/sequence_base/codec/marked.py +337 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/constants/version.py +2 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/flags.py +2 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/policy.py +3 -2
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ActRequest.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ActResponse.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ActionChunk.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ActionContract.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ActionStep.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Actions.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/AdapterPair.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/AddLocal.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Autoscaler.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/BenchmarkConditionSpec.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/BenchmarkRegisterRequest.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/BenchmarkSpec.json +2 -2
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/BenchmarkStatusResponse.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/BootstrapResponse.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Camera.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/CodeBundle.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Concurrency.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Conditions.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ConnectRequest.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ConnectResponse.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/DeploymentAccepted.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/DeploymentPolicy.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/DeploymentRequest.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/DeploymentStatusResponse.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Embodiment.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ErrorBody.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ErrorCodes.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Estimate.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/EvalDryRunResponse.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/EvalReport.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/EvalRequest.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/EvalResponse.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageFrame.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageSpec.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepAddLocal.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepApt.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepEnv.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepProject.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepRun.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepTemplateFiles.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepUvPip.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/InferParam.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Lease.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/MetricAgg.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Observation.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ObservationContract.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Observations.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ObserveRequest.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ObserveResponse.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/PolicyRegistration.json +3 -3
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/PolicySpec.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Project.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Proprioception.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/RenderSpec.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ReplaySpec.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Resources.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/RolloutResult.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/SecretCreateRequest.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/SecretInfo.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/SecretRef.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/SeqFlags.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Step.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Stream.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/TaskAgg.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/TemplateFiles.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Tensors.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Timeouts.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/UploadTarget.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Variant.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/VolumeFile.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/VolumeInfo.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/VolumeMount.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/VolumeRef.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/VolumeUploadRequest.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/VolumeUploadResponse.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/WarmingError.json +1 -1
- {sequence_base-0.4.30 → sequence_base-0.4.31}/.gitignore +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/README.md +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/__init__.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/auth.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/by_name.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/codec/__init__.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/codec/image.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/constants/__init__.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/constants/cards.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/constants/headers.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/constants/limits.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/construct.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/__init__.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/accounts.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/act.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/adapters.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/benchmarks.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/connect.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/deployments.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/errors.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/eval.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/io.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/obs_schedule.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/observe.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/own_io.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/templates.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/volumes.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/wire.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/episode.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/errors/__init__.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/errors/transport.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/feedback.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/msgpack_io.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/obs/__init__.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/obs/log.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/obs/rid.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/presets/__init__.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/presets/frames.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/presets/robot_arm.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schema_export.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/telemetry/__init__.py +0 -0
- {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/telemetry/events.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sequence-base
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.31
|
|
4
4
|
Summary: Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness.
|
|
5
5
|
Project-URL: Changelog, https://generalsequences.com/docs/changelog/
|
|
6
6
|
Requires-Python: >=3.10
|
|
@@ -8,6 +8,10 @@ Requires-Dist: pydantic>=2.0
|
|
|
8
8
|
Provides-Extra: codec
|
|
9
9
|
Requires-Dist: numpy>=1.26; extra == 'codec'
|
|
10
10
|
Requires-Dist: pillow>=10; extra == 'codec'
|
|
11
|
+
Provides-Extra: observe
|
|
12
|
+
Requires-Dist: av>=17; extra == 'observe'
|
|
13
|
+
Requires-Dist: numpy>=1.26; extra == 'observe'
|
|
14
|
+
Requires-Dist: zstandard>=0.22; extra == 'observe'
|
|
11
15
|
Provides-Extra: test
|
|
12
16
|
Requires-Dist: hypothesis>=6; extra == 'test'
|
|
13
17
|
Requires-Dist: jsonschema>=4; extra == 'test'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sequence-base"
|
|
3
|
-
version = "0.4.
|
|
3
|
+
version = "0.4.31"
|
|
4
4
|
description = "Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness."
|
|
5
5
|
requires-python = ">=3.10"
|
|
6
6
|
dependencies = ["pydantic>=2.0"]
|
|
@@ -10,6 +10,7 @@ Changelog = "https://generalsequences.com/docs/changelog/"
|
|
|
10
10
|
|
|
11
11
|
[project.optional-dependencies]
|
|
12
12
|
codec = ["pillow>=10", "numpy>=1.26"]
|
|
13
|
+
observe = ["numpy>=1.26", "av>=17", "zstandard>=0.22"]
|
|
13
14
|
test = ["pytest", "jsonschema>=4", "hypothesis>=6", "pyyaml"]
|
|
14
15
|
|
|
15
16
|
[build-system]
|
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Marked observation items: what the observation format puts on the wire. A robot marks items of its own observe data as
|
|
3
|
+
sequence_ai.Video, Depth or Tactile; each marked item travels as the map below and the worker turns it back into the
|
|
4
|
+
array it was, so the policy's observe function reads the data the robot built, item for item.
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
In one sentence: the robot's SDK keeps one Encoder per marked item of an episode (its path in the observe data) → each
|
|
9
|
+
observe() puts the map that item's encoder writes in place of the marked value → the worker keeps one Decoder per path
|
|
10
|
+
of each episode (Streams) and puts each map's array back (decode) before the policy's observe function is called.
|
|
11
|
+
|
|
12
|
+
The three kinds, as the user named them:
|
|
13
|
+
- video - an RGB camera frame, HxWx3 uint8: H.264 for low latency - no B-frames, no lookahead, every frame out of the
|
|
14
|
+
encoder as it goes in - with a keyframe every second (KEYFRAME_S) and whenever the robot asks for one (force_keyframe:
|
|
15
|
+
after a reconnect, or after frames it dropped). Lossy (crf VIDEO_CRF).
|
|
16
|
+
- depth - a depth map of any integer or float dtype, in the robot's own unit: lossless. Each frame on its own, every
|
|
17
|
+
value predicted from its left neighbour (an integer's difference, a float's bits XORed) and the bytes zstd-compressed.
|
|
18
|
+
640x480 sensor depth comes to about 2.3-2.5 times smaller in about a millisecond each way, as small as anything else
|
|
19
|
+
measured and the fastest; frames are not predicted from the frame before, because a sensor's noise is new in every
|
|
20
|
+
frame and that gained nothing.
|
|
21
|
+
- tactile - a tactile reading. Image-like (HxWx3 uint8, what a camera-based sensor gives) is H.264 at high quality
|
|
22
|
+
(crf TACTILE_CRF); any other array, a pressure grid, travels as it is. Named tactile whatever it travels as.
|
|
23
|
+
|
|
24
|
+
The map, version 1 (inside the observe body's msgpack; `data` is bin):
|
|
25
|
+
{"__seq__": "video" | "tactile", "v": 1, "codec": "h264", "w", "h", "n", "kf", "data"}
|
|
26
|
+
w, h: the frame's size (an odd one is padded to even for the encoder and cut back after); n: the frame's number in
|
|
27
|
+
its stream, from 0, never reused - a gap in n is frames the robot dropped; kf: a keyframe, which a decoder can
|
|
28
|
+
start from; data: one H.264 access unit (Annex B, its parameter sets in front of every keyframe).
|
|
29
|
+
{"__seq__": "depth", "v": 1, "codec": "zstd-delta" | "zstd-xor", "dtype", "shape", "data"}
|
|
30
|
+
dtype: numpy's string for it ("<u2", "<f4", ...); shape: the array's; data: the predicted values' bytes, byte
|
|
31
|
+
planes apart, zstd-compressed.
|
|
32
|
+
{"__seq__": "tactile", "v": 1, "codec": "raw", "value"}
|
|
33
|
+
value: the array (or whatever was marked), as msgpack carries any array.
|
|
34
|
+
|
|
35
|
+
WHY a map with a reserved key rather than a msgpack extension type: sequence_base.msgpack_io refuses every extension
|
|
36
|
+
type, so that reading a body runs nothing its sender chose; a map reads as data everywhere.
|
|
37
|
+
WHY the codecs live here: the SDK encodes with the same module the worker decodes with, so the two cannot drift, and one
|
|
38
|
+
round-trip test pins the format (tests/test_marked.py), as codec/image.py does for the frames of the robot-arm preset.
|
|
39
|
+
|
|
40
|
+
Needs numpy, PyAV (av) and zstandard - `pip install 'sequence-base[observe]'`; each is imported where a codec first
|
|
41
|
+
needs it, so nothing here is needed by a robot that marks nothing.
|
|
42
|
+
|
|
43
|
+
enc = encoder("video"); wire = enc.encode(frame); enc.force_keyframe() # the robot, one per marked item
|
|
44
|
+
streams = Streams(); value = decode(value, streams) # the worker, one Streams per episode
|
|
45
|
+
"""
|
|
46
|
+
from __future__ import annotations
|
|
47
|
+
|
|
48
|
+
import time
|
|
49
|
+
from fractions import Fraction
|
|
50
|
+
from typing import Any, Callable
|
|
51
|
+
|
|
52
|
+
__all__ = ["KINDS", "MARK", "VERSION", "Decoder", "Encoder", "MarkedError", "NeedsKeyframe", "Streams", "decode",
|
|
53
|
+
"decode_one", "encoder", "is_marked"]
|
|
54
|
+
|
|
55
|
+
MARK = "__seq__"
|
|
56
|
+
VERSION = 1
|
|
57
|
+
KINDS = ("video", "depth", "tactile")
|
|
58
|
+
|
|
59
|
+
KEYFRAME_S = 1.0
|
|
60
|
+
VIDEO_CRF = 23
|
|
61
|
+
TACTILE_CRF = 12
|
|
62
|
+
_PRESET = "ultrafast"
|
|
63
|
+
_ZSTD_LEVEL = 1
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class MarkedError(ValueError):
|
|
67
|
+
"""A value a marker cannot carry (encoding), or a map that is not one this reads (decoding)."""
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class NeedsKeyframe(MarkedError):
|
|
71
|
+
"""A video frame that follows a gap in its stream and is not a keyframe: nothing before it to decode it from. The
|
|
72
|
+
robot's encoder sends a keyframe after any frame it dropped (force_keyframe), so this is never the robot's data."""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def is_marked(value: Any) -> bool:
|
|
76
|
+
return isinstance(value, dict) and value.get(MARK) in KINDS
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _wire(kind: str, **fields: Any) -> dict:
|
|
80
|
+
"""A marked item's map: its kind under MARK, the version, then its codec's fields."""
|
|
81
|
+
return dict([(MARK, kind), ("v", VERSION), *fields.items()])
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class Encoder:
|
|
86
|
+
"""One marked item's stream: `encode(value)` -> its map for this observe; `force_keyframe()` makes the next video
|
|
87
|
+
frame a keyframe."""
|
|
88
|
+
|
|
89
|
+
kind = ""
|
|
90
|
+
|
|
91
|
+
def encode(self, value: Any) -> dict:
|
|
92
|
+
raise NotImplementedError
|
|
93
|
+
|
|
94
|
+
def force_keyframe(self) -> None:
|
|
95
|
+
"""Nothing to do for a kind with no state between frames."""
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def encoder(kind: str, *, clock: Callable[[], float] = time.monotonic) -> Encoder:
|
|
99
|
+
"""A new stream's encoder for `kind` ("video", "depth" or "tactile")."""
|
|
100
|
+
if kind == "video":
|
|
101
|
+
return _VideoEncoder(clock)
|
|
102
|
+
if kind == "depth":
|
|
103
|
+
return _DepthEncoder()
|
|
104
|
+
if kind == "tactile":
|
|
105
|
+
return _TactileEncoder(clock)
|
|
106
|
+
raise MarkedError(f"no marker {kind!r}: the kinds are {', '.join(KINDS)}")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class _H264:
|
|
110
|
+
"""libx264 for one stream: low latency, every frame out at once, a forced keyframe an IDR the decoder starts from."""
|
|
111
|
+
|
|
112
|
+
def __init__(self, crf: int, clock: Callable[[], float]) -> None:
|
|
113
|
+
self._crf, self._clock = crf, clock
|
|
114
|
+
self._ctx = None
|
|
115
|
+
self._size: tuple[int, int] | None = None
|
|
116
|
+
self._n = 0
|
|
117
|
+
self._key_at: float | None = None
|
|
118
|
+
self._force = True
|
|
119
|
+
|
|
120
|
+
def force_keyframe(self) -> None:
|
|
121
|
+
self._force = True
|
|
122
|
+
|
|
123
|
+
def encode(self, frame: Any, kind: str) -> dict:
|
|
124
|
+
import av
|
|
125
|
+
import numpy as np
|
|
126
|
+
|
|
127
|
+
a = np.asarray(frame)
|
|
128
|
+
if a.dtype != np.uint8 or a.ndim != 3 or a.shape[2] != 3 or a.shape[0] < 2 or a.shape[1] < 2:
|
|
129
|
+
raise MarkedError(f"a {kind} frame is an RGB image, HxWx3 uint8; got {a.dtype} {tuple(a.shape)} - send other "
|
|
130
|
+
"arrays unmarked, or as Depth or Tactile")
|
|
131
|
+
h, w = int(a.shape[0]), int(a.shape[1])
|
|
132
|
+
if (h, w) != self._size:
|
|
133
|
+
self._open(av, h, w)
|
|
134
|
+
if h % 2 or w % 2:
|
|
135
|
+
a = np.pad(a, ((0, h % 2), (0, w % 2), (0, 0)), mode="edge")
|
|
136
|
+
now = self._clock()
|
|
137
|
+
key = self._force or self._key_at is None or now - self._key_at >= KEYFRAME_S
|
|
138
|
+
vf = av.VideoFrame.from_ndarray(np.ascontiguousarray(a), format="rgb24").reformat(format="yuv420p")
|
|
139
|
+
vf.pts = self._n
|
|
140
|
+
if key:
|
|
141
|
+
vf.pict_type = av.video.frame.PictureType.I
|
|
142
|
+
packets = self._ctx.encode(vf)
|
|
143
|
+
if len(packets) != 1:
|
|
144
|
+
raise MarkedError(f"the H.264 encoder answered {len(packets)} packets for one frame")
|
|
145
|
+
data = bytes(packets[0])
|
|
146
|
+
kf = bool(packets[0].is_keyframe)
|
|
147
|
+
if kf:
|
|
148
|
+
self._key_at, self._force = now, False
|
|
149
|
+
n, self._n = self._n, self._n + 1
|
|
150
|
+
return _wire(kind, codec="h264", w=w, h=h, n=n, kf=kf, data=data)
|
|
151
|
+
|
|
152
|
+
def _open(self, av, h: int, w: int) -> None:
|
|
153
|
+
ctx = av.CodecContext.create("libx264", "w")
|
|
154
|
+
ctx.width, ctx.height, ctx.pix_fmt = w + w % 2, h + h % 2, "yuv420p"
|
|
155
|
+
ctx.time_base = Fraction(1, 30)
|
|
156
|
+
|
|
157
|
+
ctx.options = {"preset": _PRESET, "tune": "zerolatency", "crf": str(self._crf), "forced-idr": "1",
|
|
158
|
+
"x264-params": "keyint=1000000:min-keyint=1:scenecut=0:bframes=0"}
|
|
159
|
+
ctx.thread_count = 1
|
|
160
|
+
ctx.open()
|
|
161
|
+
self._ctx, self._size, self._force = ctx, (h, w), True
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
class _VideoEncoder(Encoder):
|
|
165
|
+
kind = "video"
|
|
166
|
+
|
|
167
|
+
def __init__(self, clock: Callable[[], float]) -> None:
|
|
168
|
+
self._h264 = _H264(VIDEO_CRF, clock)
|
|
169
|
+
|
|
170
|
+
def encode(self, value: Any) -> dict:
|
|
171
|
+
return self._h264.encode(value, "video")
|
|
172
|
+
|
|
173
|
+
def force_keyframe(self) -> None:
|
|
174
|
+
self._h264.force_keyframe()
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
class _DepthEncoder(Encoder):
|
|
178
|
+
kind = "depth"
|
|
179
|
+
|
|
180
|
+
def encode(self, value: Any) -> dict:
|
|
181
|
+
import numpy as np
|
|
182
|
+
|
|
183
|
+
a = np.asarray(value)
|
|
184
|
+
if a.dtype.kind not in "uif" or a.ndim < 1 or a.size == 0:
|
|
185
|
+
raise MarkedError(f"a depth map is a non-empty array of integers or floats; got {a.dtype} {tuple(a.shape)}")
|
|
186
|
+
codec, data = _predict(a)
|
|
187
|
+
return _wire("depth", codec=codec, dtype=a.dtype.str, shape=list(a.shape), data=data)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
class _TactileEncoder(Encoder):
|
|
191
|
+
kind = "tactile"
|
|
192
|
+
|
|
193
|
+
def __init__(self, clock: Callable[[], float]) -> None:
|
|
194
|
+
self._h264 = _H264(TACTILE_CRF, clock)
|
|
195
|
+
|
|
196
|
+
def encode(self, value: Any) -> dict:
|
|
197
|
+
import numpy as np
|
|
198
|
+
|
|
199
|
+
a = np.asarray(value) if not isinstance(value, (bytes, str, dict)) else None
|
|
200
|
+
if a is not None and a.dtype == np.uint8 and a.ndim == 3 and a.shape[2] == 3 and min(a.shape[:2]) >= 2:
|
|
201
|
+
return self._h264.encode(a, "tactile")
|
|
202
|
+
return _wire("tactile", codec="raw", value=value)
|
|
203
|
+
|
|
204
|
+
def force_keyframe(self) -> None:
|
|
205
|
+
self._h264.force_keyframe()
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _predict(a) -> tuple[str, bytes]:
|
|
210
|
+
import numpy as np
|
|
211
|
+
import zstandard
|
|
212
|
+
|
|
213
|
+
le = a.astype(a.dtype.newbyteorder("<"), copy=False)
|
|
214
|
+
size = le.dtype.itemsize
|
|
215
|
+
rows = np.ascontiguousarray(le).reshape(-1, le.shape[-1]).view(f"<u{size}")
|
|
216
|
+
out = rows.copy()
|
|
217
|
+
if le.dtype.kind == "f":
|
|
218
|
+
out[:, 1:] = rows[:, 1:] ^ rows[:, :-1]
|
|
219
|
+
codec = "zstd-xor"
|
|
220
|
+
else:
|
|
221
|
+
out[:, 1:] = rows[:, 1:] - rows[:, :-1]
|
|
222
|
+
signed = out.view(f"<i{size}")
|
|
223
|
+
out = ((signed << 1) ^ (signed >> (8 * size - 1))).view(f"<u{size}")
|
|
224
|
+
codec = "zstd-delta"
|
|
225
|
+
planes = out.view(np.uint8).reshape(-1, size).T.tobytes()
|
|
226
|
+
return codec, zstandard.ZstdCompressor(level=_ZSTD_LEVEL).compress(planes)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _unpredict(m: dict):
|
|
230
|
+
import numpy as np
|
|
231
|
+
import zstandard
|
|
232
|
+
|
|
233
|
+
dtype, shape, codec = np.dtype(m["dtype"]), tuple(int(s) for s in m["shape"]), m["codec"]
|
|
234
|
+
size = dtype.itemsize
|
|
235
|
+
cols = shape[-1]
|
|
236
|
+
raw = zstandard.ZstdDecompressor().decompress(m["data"])
|
|
237
|
+
count = int(np.prod(shape))
|
|
238
|
+
if len(raw) != count * size or codec not in ("zstd-delta", "zstd-xor"):
|
|
239
|
+
raise MarkedError(f"a depth map that does not read back: {codec} {m['dtype']} {list(shape)}")
|
|
240
|
+
rows = np.frombuffer(raw, np.uint8).reshape(size, -1).T.copy().view(f"<u{size}").reshape(-1, cols)
|
|
241
|
+
if codec == "zstd-xor":
|
|
242
|
+
values = np.bitwise_xor.accumulate(rows, axis=1)
|
|
243
|
+
else:
|
|
244
|
+
signed = rows.view(f"<i{size}")
|
|
245
|
+
steps = ((rows >> 1).view(f"<i{size}") ^ -(signed & 1)).view(f"<u{size}")
|
|
246
|
+
values = np.cumsum(steps, axis=1, dtype=f"<u{size}")
|
|
247
|
+
return values.view(dtype.newbyteorder("<")).reshape(shape).astype(dtype, copy=False)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
class Decoder:
|
|
252
|
+
"""One marked item's stream: `decode(map)` -> the array it carries."""
|
|
253
|
+
|
|
254
|
+
def decode(self, m: dict) -> Any:
|
|
255
|
+
return decode_one(m)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
class _VideoDecoder(Decoder):
|
|
259
|
+
"""H.264 for one stream, frames in order; after a gap in n it waits for a keyframe (NeedsKeyframe until then)."""
|
|
260
|
+
|
|
261
|
+
def __init__(self) -> None:
|
|
262
|
+
self._ctx = None
|
|
263
|
+
self._last: int | None = None
|
|
264
|
+
self._whole = False
|
|
265
|
+
|
|
266
|
+
def decode(self, m: dict) -> Any:
|
|
267
|
+
import av
|
|
268
|
+
|
|
269
|
+
n, kf = int(m["n"]), bool(m["kf"])
|
|
270
|
+
if self._last is not None and n != self._last + 1:
|
|
271
|
+
self._whole = False
|
|
272
|
+
self._last = n
|
|
273
|
+
if kf:
|
|
274
|
+
self._whole = True
|
|
275
|
+
if not self._whole:
|
|
276
|
+
raise NeedsKeyframe(f"{m.get(MARK)} frame {n} follows a gap in its stream and is not a keyframe")
|
|
277
|
+
if self._ctx is None:
|
|
278
|
+
self._ctx = av.CodecContext.create("h264", "r")
|
|
279
|
+
self._ctx.thread_count = 1
|
|
280
|
+
frames = self._ctx.decode(av.Packet(m["data"]))
|
|
281
|
+
if not frames:
|
|
282
|
+
raise MarkedError(f"{m.get(MARK)} frame {n} decoded to nothing")
|
|
283
|
+
import numpy as np
|
|
284
|
+
|
|
285
|
+
rgb = frames[-1].to_ndarray(format="rgb24")
|
|
286
|
+
h, w = int(m["h"]), int(m["w"])
|
|
287
|
+
return rgb if rgb.shape[:2] == (h, w) else np.ascontiguousarray(rgb[:h, :w])
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def decode_one(m: dict) -> Any:
|
|
291
|
+
"""A map with no state between frames - depth, or tactile as it is - back to its value. Video needs its stream's
|
|
292
|
+
Decoder (Streams)."""
|
|
293
|
+
if not is_marked(m) or int(m.get("v") or 0) != VERSION:
|
|
294
|
+
raise MarkedError(f"not a marked item of version {VERSION}: {sorted(m) if isinstance(m, dict) else type(m)}")
|
|
295
|
+
codec = m.get("codec")
|
|
296
|
+
if m[MARK] == "depth":
|
|
297
|
+
return _unpredict(m)
|
|
298
|
+
if m[MARK] == "tactile" and codec == "raw":
|
|
299
|
+
return m.get("value")
|
|
300
|
+
raise MarkedError(f"a {m[MARK]} item in {codec!r} decodes in its stream's order (Streams)")
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
class Streams:
|
|
304
|
+
"""One episode's decoders, by the path of each marked item in its observe data."""
|
|
305
|
+
|
|
306
|
+
def __init__(self) -> None:
|
|
307
|
+
self._decoders: dict[tuple, Decoder] = {}
|
|
308
|
+
|
|
309
|
+
def decoder(self, path: tuple, m: dict) -> Decoder:
|
|
310
|
+
d = self._decoders.get(path)
|
|
311
|
+
if d is None:
|
|
312
|
+
d = self._decoders[path] = _VideoDecoder() if m.get("codec") == "h264" else Decoder()
|
|
313
|
+
return d
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def decode(value: Any, streams: Streams, path: tuple = ()) -> Any:
|
|
317
|
+
"""`value` with every marked map in it - in dicts and lists, at any depth - replaced by its array; everything else
|
|
318
|
+
as it was, the same objects. A map that is not marked is walked; one that is decodes with its path's decoder."""
|
|
319
|
+
if isinstance(value, dict):
|
|
320
|
+
if is_marked(value):
|
|
321
|
+
return streams.decoder(path, value).decode(value)
|
|
322
|
+
if not any(_holds_mark(v) for v in value.values()):
|
|
323
|
+
return value
|
|
324
|
+
return {k: decode(v, streams, path + (k,)) for k, v in value.items()}
|
|
325
|
+
if isinstance(value, list):
|
|
326
|
+
if not any(_holds_mark(v) for v in value):
|
|
327
|
+
return value
|
|
328
|
+
return [decode(v, streams, path + (i,)) for i, v in enumerate(value)]
|
|
329
|
+
return value
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _holds_mark(value: Any) -> bool:
|
|
333
|
+
if isinstance(value, dict):
|
|
334
|
+
return is_marked(value) or any(_holds_mark(v) for v in value.values())
|
|
335
|
+
if isinstance(value, list):
|
|
336
|
+
return any(_holds_mark(v) for v in value)
|
|
337
|
+
return False
|
|
@@ -516,8 +516,9 @@ class PolicyRegistration(BaseModel):
|
|
|
516
516
|
act carries, before INFER is called with none), RESET (an episode's first act) and ON_EPISODE (its end).
|
|
517
517
|
|
|
518
518
|
Any policy may bind SAMPLE (contract 0.5.24, `@seq.policy(sample="<method>")`): the method returning one episode's
|
|
519
|
-
|
|
520
|
-
|
|
519
|
+
calls in order, as a robot makes them - ("observe", x) for its observe(x) and ("act",) for its act(), which carries
|
|
520
|
+
nothing, every input going through observe - what its build check plays as the robot's own calls go, and its worker
|
|
521
|
+
times its infer on (the acts) when no contract says what it reads."""
|
|
521
522
|
|
|
522
523
|
model_config = ConfigDict(extra="forbid")
|
|
523
524
|
|
|
@@ -471,7 +471,7 @@
|
|
|
471
471
|
],
|
|
472
472
|
"title": "SeqFlags",
|
|
473
473
|
"type": "integer",
|
|
474
|
-
"x-contract-version": "0.5.
|
|
474
|
+
"x-contract-version": "0.5.28",
|
|
475
475
|
"x-flag-names": {
|
|
476
476
|
"CHECK": 65536,
|
|
477
477
|
"INFER": 16,
|
|
@@ -722,5 +722,5 @@
|
|
|
722
722
|
],
|
|
723
723
|
"title": "BenchmarkSpec",
|
|
724
724
|
"type": "object",
|
|
725
|
-
"x-contract-version": "0.5.
|
|
725
|
+
"x-contract-version": "0.5.28"
|
|
726
726
|
}
|