sequence-base 0.4.30__tar.gz → 0.4.31__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. {sequence_base-0.4.30 → sequence_base-0.4.31}/PKG-INFO +5 -1
  2. {sequence_base-0.4.30 → sequence_base-0.4.31}/pyproject.toml +2 -1
  3. sequence_base-0.4.31/sequence_base/codec/marked.py +337 -0
  4. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/constants/version.py +2 -1
  5. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/flags.py +2 -0
  6. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/policy.py +3 -2
  7. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ActRequest.json +1 -1
  8. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ActResponse.json +1 -1
  9. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ActionChunk.json +1 -1
  10. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ActionContract.json +1 -1
  11. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ActionStep.json +1 -1
  12. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Actions.json +1 -1
  13. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/AdapterPair.json +1 -1
  14. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/AddLocal.json +1 -1
  15. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Autoscaler.json +1 -1
  16. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/BenchmarkConditionSpec.json +1 -1
  17. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/BenchmarkRegisterRequest.json +1 -1
  18. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/BenchmarkSpec.json +2 -2
  19. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/BenchmarkStatusResponse.json +1 -1
  20. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/BootstrapResponse.json +1 -1
  21. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Camera.json +1 -1
  22. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/CodeBundle.json +1 -1
  23. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Concurrency.json +1 -1
  24. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Conditions.json +1 -1
  25. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ConnectRequest.json +1 -1
  26. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ConnectResponse.json +1 -1
  27. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/DeploymentAccepted.json +1 -1
  28. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/DeploymentPolicy.json +1 -1
  29. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/DeploymentRequest.json +1 -1
  30. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/DeploymentStatusResponse.json +1 -1
  31. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Embodiment.json +1 -1
  32. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ErrorBody.json +1 -1
  33. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ErrorCodes.json +1 -1
  34. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Estimate.json +1 -1
  35. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/EvalDryRunResponse.json +1 -1
  36. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/EvalReport.json +1 -1
  37. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/EvalRequest.json +1 -1
  38. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/EvalResponse.json +1 -1
  39. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageFrame.json +1 -1
  40. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageSpec.json +1 -1
  41. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepAddLocal.json +1 -1
  42. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepApt.json +1 -1
  43. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepEnv.json +1 -1
  44. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepProject.json +1 -1
  45. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepRun.json +1 -1
  46. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepTemplateFiles.json +1 -1
  47. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepUvPip.json +1 -1
  48. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/InferParam.json +1 -1
  49. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Lease.json +1 -1
  50. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/MetricAgg.json +1 -1
  51. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Observation.json +1 -1
  52. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ObservationContract.json +1 -1
  53. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Observations.json +1 -1
  54. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ObserveRequest.json +1 -1
  55. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ObserveResponse.json +1 -1
  56. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/PolicyRegistration.json +3 -3
  57. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/PolicySpec.json +1 -1
  58. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Project.json +1 -1
  59. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Proprioception.json +1 -1
  60. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/RenderSpec.json +1 -1
  61. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/ReplaySpec.json +1 -1
  62. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Resources.json +1 -1
  63. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/RolloutResult.json +1 -1
  64. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/SecretCreateRequest.json +1 -1
  65. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/SecretInfo.json +1 -1
  66. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/SecretRef.json +1 -1
  67. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/SeqFlags.json +1 -1
  68. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Step.json +1 -1
  69. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Stream.json +1 -1
  70. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/TaskAgg.json +1 -1
  71. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/TemplateFiles.json +1 -1
  72. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Tensors.json +1 -1
  73. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Timeouts.json +1 -1
  74. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/UploadTarget.json +1 -1
  75. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/Variant.json +1 -1
  76. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/VolumeFile.json +1 -1
  77. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/VolumeInfo.json +1 -1
  78. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/VolumeMount.json +1 -1
  79. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/VolumeRef.json +1 -1
  80. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/VolumeUploadRequest.json +1 -1
  81. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/VolumeUploadResponse.json +1 -1
  82. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schemas/WarmingError.json +1 -1
  83. {sequence_base-0.4.30 → sequence_base-0.4.31}/.gitignore +0 -0
  84. {sequence_base-0.4.30 → sequence_base-0.4.31}/README.md +0 -0
  85. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/__init__.py +0 -0
  86. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/auth.py +0 -0
  87. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/by_name.py +0 -0
  88. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/codec/__init__.py +0 -0
  89. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/codec/image.py +0 -0
  90. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/constants/__init__.py +0 -0
  91. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/constants/cards.py +0 -0
  92. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/constants/headers.py +0 -0
  93. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/constants/limits.py +0 -0
  94. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/construct.py +0 -0
  95. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/__init__.py +0 -0
  96. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/accounts.py +0 -0
  97. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/act.py +0 -0
  98. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/adapters.py +0 -0
  99. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/benchmarks.py +0 -0
  100. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/connect.py +0 -0
  101. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/deployments.py +0 -0
  102. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/errors.py +0 -0
  103. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/eval.py +0 -0
  104. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/io.py +0 -0
  105. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/obs_schedule.py +0 -0
  106. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/observe.py +0 -0
  107. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/own_io.py +0 -0
  108. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/templates.py +0 -0
  109. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/volumes.py +0 -0
  110. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/contract/wire.py +0 -0
  111. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/episode.py +0 -0
  112. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/errors/__init__.py +0 -0
  113. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/errors/transport.py +0 -0
  114. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/feedback.py +0 -0
  115. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/msgpack_io.py +0 -0
  116. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/obs/__init__.py +0 -0
  117. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/obs/log.py +0 -0
  118. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/obs/rid.py +0 -0
  119. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/presets/__init__.py +0 -0
  120. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/presets/frames.py +0 -0
  121. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/presets/robot_arm.py +0 -0
  122. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/schema_export.py +0 -0
  123. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/telemetry/__init__.py +0 -0
  124. {sequence_base-0.4.30 → sequence_base-0.4.31}/sequence_base/telemetry/events.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: sequence-base
3
- Version: 0.4.30
3
+ Version: 0.4.31
4
4
  Summary: Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness.
5
5
  Project-URL: Changelog, https://generalsequences.com/docs/changelog/
6
6
  Requires-Python: >=3.10
@@ -8,6 +8,10 @@ Requires-Dist: pydantic>=2.0
8
8
  Provides-Extra: codec
9
9
  Requires-Dist: numpy>=1.26; extra == 'codec'
10
10
  Requires-Dist: pillow>=10; extra == 'codec'
11
+ Provides-Extra: observe
12
+ Requires-Dist: av>=17; extra == 'observe'
13
+ Requires-Dist: numpy>=1.26; extra == 'observe'
14
+ Requires-Dist: zstandard>=0.22; extra == 'observe'
11
15
  Provides-Extra: test
12
16
  Requires-Dist: hypothesis>=6; extra == 'test'
13
17
  Requires-Dist: jsonschema>=4; extra == 'test'
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "sequence-base"
3
- version = "0.4.30"
3
+ version = "0.4.31"
4
4
  description = "Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness."
5
5
  requires-python = ">=3.10"
6
6
  dependencies = ["pydantic>=2.0"]
@@ -10,6 +10,7 @@ Changelog = "https://generalsequences.com/docs/changelog/"
10
10
 
11
11
  [project.optional-dependencies]
12
12
  codec = ["pillow>=10", "numpy>=1.26"]
13
+ observe = ["numpy>=1.26", "av>=17", "zstandard>=0.22"]
13
14
  test = ["pytest", "jsonschema>=4", "hypothesis>=6", "pyyaml"]
14
15
 
15
16
  [build-system]
@@ -0,0 +1,337 @@
1
+ """
2
+ Marked observation items: what the observation format puts on the wire. A robot marks items of its own observe data as
3
+ sequence_ai.Video, Depth or Tactile; each marked item travels as the map below and the worker turns it back into the
4
+ array it was, so the policy's observe function reads the data the robot built, item for item.
5
+
6
+
7
+
8
+ In one sentence: the robot's SDK keeps one Encoder per marked item of an episode (its path in the observe data) → each
9
+ observe() puts the map that item's encoder writes in place of the marked value → the worker keeps one Decoder per path
10
+ of each episode (Streams) and puts each map's array back (decode) before the policy's observe function is called.
11
+
12
+ The three kinds, as the user named them:
13
+ - video - an RGB camera frame, HxWx3 uint8: H.264 for low latency - no B-frames, no lookahead, every frame out of the
14
+ encoder as it goes in - with a keyframe every second (KEYFRAME_S) and whenever the robot asks for one (force_keyframe:
15
+ after a reconnect, or after frames it dropped). Lossy (crf VIDEO_CRF).
16
+ - depth - a depth map of any integer or float dtype, in the robot's own unit: lossless. Each frame on its own, every
17
+ value predicted from its left neighbour (an integer's difference, a float's bits XORed) and the bytes zstd-compressed.
18
+ 640x480 sensor depth comes to about 2.3-2.5 times smaller in about a millisecond each way, as small as anything else
19
+ measured and the fastest; frames are not predicted from the frame before, because a sensor's noise is new in every
20
+ frame and that gained nothing.
21
+ - tactile - a tactile reading. Image-like (HxWx3 uint8, what a camera-based sensor gives) is H.264 at high quality
22
+ (crf TACTILE_CRF); any other array, a pressure grid, travels as it is. Named tactile whatever it travels as.
23
+
24
+ The map, version 1 (inside the observe body's msgpack; `data` is bin):
25
+ {"__seq__": "video" | "tactile", "v": 1, "codec": "h264", "w", "h", "n", "kf", "data"}
26
+ w, h: the frame's size (an odd one is padded to even for the encoder and cut back after); n: the frame's number in
27
+ its stream, from 0, never reused - a gap in n is frames the robot dropped; kf: a keyframe, which a decoder can
28
+ start from; data: one H.264 access unit (Annex B, its parameter sets in front of every keyframe).
29
+ {"__seq__": "depth", "v": 1, "codec": "zstd-delta" | "zstd-xor", "dtype", "shape", "data"}
30
+ dtype: numpy's string for it ("<u2", "<f4", ...); shape: the array's; data: the predicted values' bytes, byte
31
+ planes apart, zstd-compressed.
32
+ {"__seq__": "tactile", "v": 1, "codec": "raw", "value"}
33
+ value: the array (or whatever was marked), as msgpack carries any array.
34
+
35
+ WHY a map with a reserved key rather than a msgpack extension type: sequence_base.msgpack_io refuses every extension
36
+ type, so that reading a body runs nothing its sender chose; a map reads as data everywhere.
37
+ WHY the codecs live here: the SDK encodes with the same module the worker decodes with, so the two cannot drift, and one
38
+ round-trip test pins the format (tests/test_marked.py), as codec/image.py does for the frames of the robot-arm preset.
39
+
40
+ Needs numpy, PyAV (av) and zstandard - `pip install 'sequence-base[observe]'`; each is imported where a codec first
41
+ needs it, so nothing here is needed by a robot that marks nothing.
42
+
43
+ enc = encoder("video"); wire = enc.encode(frame); enc.force_keyframe() # the robot, one per marked item
44
+ streams = Streams(); value = decode(value, streams) # the worker, one Streams per episode
45
+ """
46
+ from __future__ import annotations
47
+
48
+ import time
49
+ from fractions import Fraction
50
+ from typing import Any, Callable
51
+
52
+ __all__ = ["KINDS", "MARK", "VERSION", "Decoder", "Encoder", "MarkedError", "NeedsKeyframe", "Streams", "decode",
53
+ "decode_one", "encoder", "is_marked"]
54
+
55
+ MARK = "__seq__"
56
+ VERSION = 1
57
+ KINDS = ("video", "depth", "tactile")
58
+
59
+ KEYFRAME_S = 1.0
60
+ VIDEO_CRF = 23
61
+ TACTILE_CRF = 12
62
+ _PRESET = "ultrafast"
63
+ _ZSTD_LEVEL = 1
64
+
65
+
66
+ class MarkedError(ValueError):
67
+ """A value a marker cannot carry (encoding), or a map that is not one this reads (decoding)."""
68
+
69
+
70
+ class NeedsKeyframe(MarkedError):
71
+ """A video frame that follows a gap in its stream and is not a keyframe: nothing before it to decode it from. The
72
+ robot's encoder sends a keyframe after any frame it dropped (force_keyframe), so this is never the robot's data."""
73
+
74
+
75
+ def is_marked(value: Any) -> bool:
76
+ return isinstance(value, dict) and value.get(MARK) in KINDS
77
+
78
+
79
+ def _wire(kind: str, **fields: Any) -> dict:
80
+ """A marked item's map: its kind under MARK, the version, then its codec's fields."""
81
+ return dict([(MARK, kind), ("v", VERSION), *fields.items()])
82
+
83
+
84
+
85
+ class Encoder:
86
+ """One marked item's stream: `encode(value)` -> its map for this observe; `force_keyframe()` makes the next video
87
+ frame a keyframe."""
88
+
89
+ kind = ""
90
+
91
+ def encode(self, value: Any) -> dict:
92
+ raise NotImplementedError
93
+
94
+ def force_keyframe(self) -> None:
95
+ """Nothing to do for a kind with no state between frames."""
96
+
97
+
98
+ def encoder(kind: str, *, clock: Callable[[], float] = time.monotonic) -> Encoder:
99
+ """A new stream's encoder for `kind` ("video", "depth" or "tactile")."""
100
+ if kind == "video":
101
+ return _VideoEncoder(clock)
102
+ if kind == "depth":
103
+ return _DepthEncoder()
104
+ if kind == "tactile":
105
+ return _TactileEncoder(clock)
106
+ raise MarkedError(f"no marker {kind!r}: the kinds are {', '.join(KINDS)}")
107
+
108
+
109
+ class _H264:
110
+ """libx264 for one stream: low latency, every frame out at once, a forced keyframe an IDR the decoder starts from."""
111
+
112
+ def __init__(self, crf: int, clock: Callable[[], float]) -> None:
113
+ self._crf, self._clock = crf, clock
114
+ self._ctx = None
115
+ self._size: tuple[int, int] | None = None
116
+ self._n = 0
117
+ self._key_at: float | None = None
118
+ self._force = True
119
+
120
+ def force_keyframe(self) -> None:
121
+ self._force = True
122
+
123
+ def encode(self, frame: Any, kind: str) -> dict:
124
+ import av
125
+ import numpy as np
126
+
127
+ a = np.asarray(frame)
128
+ if a.dtype != np.uint8 or a.ndim != 3 or a.shape[2] != 3 or a.shape[0] < 2 or a.shape[1] < 2:
129
+ raise MarkedError(f"a {kind} frame is an RGB image, HxWx3 uint8; got {a.dtype} {tuple(a.shape)} - send other "
130
+ "arrays unmarked, or as Depth or Tactile")
131
+ h, w = int(a.shape[0]), int(a.shape[1])
132
+ if (h, w) != self._size:
133
+ self._open(av, h, w)
134
+ if h % 2 or w % 2:
135
+ a = np.pad(a, ((0, h % 2), (0, w % 2), (0, 0)), mode="edge")
136
+ now = self._clock()
137
+ key = self._force or self._key_at is None or now - self._key_at >= KEYFRAME_S
138
+ vf = av.VideoFrame.from_ndarray(np.ascontiguousarray(a), format="rgb24").reformat(format="yuv420p")
139
+ vf.pts = self._n
140
+ if key:
141
+ vf.pict_type = av.video.frame.PictureType.I
142
+ packets = self._ctx.encode(vf)
143
+ if len(packets) != 1:
144
+ raise MarkedError(f"the H.264 encoder answered {len(packets)} packets for one frame")
145
+ data = bytes(packets[0])
146
+ kf = bool(packets[0].is_keyframe)
147
+ if kf:
148
+ self._key_at, self._force = now, False
149
+ n, self._n = self._n, self._n + 1
150
+ return _wire(kind, codec="h264", w=w, h=h, n=n, kf=kf, data=data)
151
+
152
+ def _open(self, av, h: int, w: int) -> None:
153
+ ctx = av.CodecContext.create("libx264", "w")
154
+ ctx.width, ctx.height, ctx.pix_fmt = w + w % 2, h + h % 2, "yuv420p"
155
+ ctx.time_base = Fraction(1, 30)
156
+
157
+ ctx.options = {"preset": _PRESET, "tune": "zerolatency", "crf": str(self._crf), "forced-idr": "1",
158
+ "x264-params": "keyint=1000000:min-keyint=1:scenecut=0:bframes=0"}
159
+ ctx.thread_count = 1
160
+ ctx.open()
161
+ self._ctx, self._size, self._force = ctx, (h, w), True
162
+
163
+
164
+ class _VideoEncoder(Encoder):
165
+ kind = "video"
166
+
167
+ def __init__(self, clock: Callable[[], float]) -> None:
168
+ self._h264 = _H264(VIDEO_CRF, clock)
169
+
170
+ def encode(self, value: Any) -> dict:
171
+ return self._h264.encode(value, "video")
172
+
173
+ def force_keyframe(self) -> None:
174
+ self._h264.force_keyframe()
175
+
176
+
177
+ class _DepthEncoder(Encoder):
178
+ kind = "depth"
179
+
180
+ def encode(self, value: Any) -> dict:
181
+ import numpy as np
182
+
183
+ a = np.asarray(value)
184
+ if a.dtype.kind not in "uif" or a.ndim < 1 or a.size == 0:
185
+ raise MarkedError(f"a depth map is a non-empty array of integers or floats; got {a.dtype} {tuple(a.shape)}")
186
+ codec, data = _predict(a)
187
+ return _wire("depth", codec=codec, dtype=a.dtype.str, shape=list(a.shape), data=data)
188
+
189
+
190
+ class _TactileEncoder(Encoder):
191
+ kind = "tactile"
192
+
193
+ def __init__(self, clock: Callable[[], float]) -> None:
194
+ self._h264 = _H264(TACTILE_CRF, clock)
195
+
196
+ def encode(self, value: Any) -> dict:
197
+ import numpy as np
198
+
199
+ a = np.asarray(value) if not isinstance(value, (bytes, str, dict)) else None
200
+ if a is not None and a.dtype == np.uint8 and a.ndim == 3 and a.shape[2] == 3 and min(a.shape[:2]) >= 2:
201
+ return self._h264.encode(a, "tactile")
202
+ return _wire("tactile", codec="raw", value=value)
203
+
204
+ def force_keyframe(self) -> None:
205
+ self._h264.force_keyframe()
206
+
207
+
208
+
209
+ def _predict(a) -> tuple[str, bytes]:
210
+ import numpy as np
211
+ import zstandard
212
+
213
+ le = a.astype(a.dtype.newbyteorder("<"), copy=False)
214
+ size = le.dtype.itemsize
215
+ rows = np.ascontiguousarray(le).reshape(-1, le.shape[-1]).view(f"<u{size}")
216
+ out = rows.copy()
217
+ if le.dtype.kind == "f":
218
+ out[:, 1:] = rows[:, 1:] ^ rows[:, :-1]
219
+ codec = "zstd-xor"
220
+ else:
221
+ out[:, 1:] = rows[:, 1:] - rows[:, :-1]
222
+ signed = out.view(f"<i{size}")
223
+ out = ((signed << 1) ^ (signed >> (8 * size - 1))).view(f"<u{size}")
224
+ codec = "zstd-delta"
225
+ planes = out.view(np.uint8).reshape(-1, size).T.tobytes()
226
+ return codec, zstandard.ZstdCompressor(level=_ZSTD_LEVEL).compress(planes)
227
+
228
+
229
+ def _unpredict(m: dict):
230
+ import numpy as np
231
+ import zstandard
232
+
233
+ dtype, shape, codec = np.dtype(m["dtype"]), tuple(int(s) for s in m["shape"]), m["codec"]
234
+ size = dtype.itemsize
235
+ cols = shape[-1]
236
+ raw = zstandard.ZstdDecompressor().decompress(m["data"])
237
+ count = int(np.prod(shape))
238
+ if len(raw) != count * size or codec not in ("zstd-delta", "zstd-xor"):
239
+ raise MarkedError(f"a depth map that does not read back: {codec} {m['dtype']} {list(shape)}")
240
+ rows = np.frombuffer(raw, np.uint8).reshape(size, -1).T.copy().view(f"<u{size}").reshape(-1, cols)
241
+ if codec == "zstd-xor":
242
+ values = np.bitwise_xor.accumulate(rows, axis=1)
243
+ else:
244
+ signed = rows.view(f"<i{size}")
245
+ steps = ((rows >> 1).view(f"<i{size}") ^ -(signed & 1)).view(f"<u{size}")
246
+ values = np.cumsum(steps, axis=1, dtype=f"<u{size}")
247
+ return values.view(dtype.newbyteorder("<")).reshape(shape).astype(dtype, copy=False)
248
+
249
+
250
+
251
+ class Decoder:
252
+ """One marked item's stream: `decode(map)` -> the array it carries."""
253
+
254
+ def decode(self, m: dict) -> Any:
255
+ return decode_one(m)
256
+
257
+
258
+ class _VideoDecoder(Decoder):
259
+ """H.264 for one stream, frames in order; after a gap in n it waits for a keyframe (NeedsKeyframe until then)."""
260
+
261
+ def __init__(self) -> None:
262
+ self._ctx = None
263
+ self._last: int | None = None
264
+ self._whole = False
265
+
266
+ def decode(self, m: dict) -> Any:
267
+ import av
268
+
269
+ n, kf = int(m["n"]), bool(m["kf"])
270
+ if self._last is not None and n != self._last + 1:
271
+ self._whole = False
272
+ self._last = n
273
+ if kf:
274
+ self._whole = True
275
+ if not self._whole:
276
+ raise NeedsKeyframe(f"{m.get(MARK)} frame {n} follows a gap in its stream and is not a keyframe")
277
+ if self._ctx is None:
278
+ self._ctx = av.CodecContext.create("h264", "r")
279
+ self._ctx.thread_count = 1
280
+ frames = self._ctx.decode(av.Packet(m["data"]))
281
+ if not frames:
282
+ raise MarkedError(f"{m.get(MARK)} frame {n} decoded to nothing")
283
+ import numpy as np
284
+
285
+ rgb = frames[-1].to_ndarray(format="rgb24")
286
+ h, w = int(m["h"]), int(m["w"])
287
+ return rgb if rgb.shape[:2] == (h, w) else np.ascontiguousarray(rgb[:h, :w])
288
+
289
+
290
+ def decode_one(m: dict) -> Any:
291
+ """A map with no state between frames - depth, or tactile as it is - back to its value. Video needs its stream's
292
+ Decoder (Streams)."""
293
+ if not is_marked(m) or int(m.get("v") or 0) != VERSION:
294
+ raise MarkedError(f"not a marked item of version {VERSION}: {sorted(m) if isinstance(m, dict) else type(m)}")
295
+ codec = m.get("codec")
296
+ if m[MARK] == "depth":
297
+ return _unpredict(m)
298
+ if m[MARK] == "tactile" and codec == "raw":
299
+ return m.get("value")
300
+ raise MarkedError(f"a {m[MARK]} item in {codec!r} decodes in its stream's order (Streams)")
301
+
302
+
303
+ class Streams:
304
+ """One episode's decoders, by the path of each marked item in its observe data."""
305
+
306
+ def __init__(self) -> None:
307
+ self._decoders: dict[tuple, Decoder] = {}
308
+
309
+ def decoder(self, path: tuple, m: dict) -> Decoder:
310
+ d = self._decoders.get(path)
311
+ if d is None:
312
+ d = self._decoders[path] = _VideoDecoder() if m.get("codec") == "h264" else Decoder()
313
+ return d
314
+
315
+
316
+ def decode(value: Any, streams: Streams, path: tuple = ()) -> Any:
317
+ """`value` with every marked map in it - in dicts and lists, at any depth - replaced by its array; everything else
318
+ as it was, the same objects. A map that is not marked is walked; one that is decodes with its path's decoder."""
319
+ if isinstance(value, dict):
320
+ if is_marked(value):
321
+ return streams.decoder(path, value).decode(value)
322
+ if not any(_holds_mark(v) for v in value.values()):
323
+ return value
324
+ return {k: decode(v, streams, path + (k,)) for k, v in value.items()}
325
+ if isinstance(value, list):
326
+ if not any(_holds_mark(v) for v in value):
327
+ return value
328
+ return [decode(v, streams, path + (i,)) for i, v in enumerate(value)]
329
+ return value
330
+
331
+
332
+ def _holds_mark(value: Any) -> bool:
333
+ if isinstance(value, dict):
334
+ return is_marked(value) or any(_holds_mark(v) for v in value.values())
335
+ if isinstance(value, list):
336
+ return any(_holds_mark(v) for v in value)
337
+ return False
@@ -4,7 +4,8 @@
4
4
 
5
5
 
6
6
 
7
- CONTRACT_VERSION = "0.5.27"
7
+ CONTRACT_VERSION = "0.5.28"
8
+
8
9
 
9
10
 
10
11
 
@@ -48,6 +48,8 @@ class SeqFlags(IntFlag):
48
48
 
49
49
 
50
50
 
51
+
52
+
51
53
  SAMPLE = 524288
52
54
 
53
55
 
@@ -516,8 +516,9 @@ class PolicyRegistration(BaseModel):
516
516
  act carries, before INFER is called with none), RESET (an episode's first act) and ON_EPISODE (its end).
517
517
 
518
518
  Any policy may bind SAMPLE (contract 0.5.24, `@seq.policy(sample="<method>")`): the method returning one episode's
519
- act inputs, in order, each as an act carries it - what its build check acts with and its worker times its infer on
520
- when no contract says what it reads."""
519
+ calls in order, as a robot makes them - ("observe", x) for its observe(x) and ("act",) for its act(), which carries
520
+ nothing, every input going through observe - what its build check plays as the robot's own calls go, and its worker
521
+ times its infer on (the acts) when no contract says what it reads."""
521
522
 
522
523
  model_config = ConfigDict(extra="forbid")
523
524
 
@@ -107,5 +107,5 @@
107
107
  ],
108
108
  "title": "ActRequest",
109
109
  "type": "object",
110
- "x-contract-version": "0.5.27"
110
+ "x-contract-version": "0.5.28"
111
111
  }
@@ -89,5 +89,5 @@
89
89
  ],
90
90
  "title": "ActResponse",
91
91
  "type": "object",
92
- "x-contract-version": "0.5.27"
92
+ "x-contract-version": "0.5.28"
93
93
  }
@@ -99,5 +99,5 @@
99
99
  ],
100
100
  "title": "ActionChunk",
101
101
  "type": "object",
102
- "x-contract-version": "0.5.27"
102
+ "x-contract-version": "0.5.28"
103
103
  }
@@ -96,5 +96,5 @@
96
96
  ],
97
97
  "title": "ActionContract",
98
98
  "type": "object",
99
- "x-contract-version": "0.5.27"
99
+ "x-contract-version": "0.5.28"
100
100
  }
@@ -19,5 +19,5 @@
19
19
  ],
20
20
  "title": "ActionStep",
21
21
  "type": "object",
22
- "x-contract-version": "0.5.27"
22
+ "x-contract-version": "0.5.28"
23
23
  }
@@ -59,5 +59,5 @@
59
59
  },
60
60
  "title": "Actions",
61
61
  "type": "object",
62
- "x-contract-version": "0.5.27"
62
+ "x-contract-version": "0.5.28"
63
63
  }
@@ -17,5 +17,5 @@
17
17
  ],
18
18
  "title": "AdapterPair",
19
19
  "type": "object",
20
- "x-contract-version": "0.5.27"
20
+ "x-contract-version": "0.5.28"
21
21
  }
@@ -23,5 +23,5 @@
23
23
  ],
24
24
  "title": "AddLocal",
25
25
  "type": "object",
26
- "x-contract-version": "0.5.27"
26
+ "x-contract-version": "0.5.28"
27
27
  }
@@ -57,5 +57,5 @@
57
57
  },
58
58
  "title": "Autoscaler",
59
59
  "type": "object",
60
- "x-contract-version": "0.5.27"
60
+ "x-contract-version": "0.5.28"
61
61
  }
@@ -50,5 +50,5 @@
50
50
  ],
51
51
  "title": "BenchmarkConditionSpec",
52
52
  "type": "object",
53
- "x-contract-version": "0.5.27"
53
+ "x-contract-version": "0.5.28"
54
54
  }
@@ -1089,5 +1089,5 @@
1089
1089
  ],
1090
1090
  "title": "BenchmarkRegisterRequest",
1091
1091
  "type": "object",
1092
- "x-contract-version": "0.5.27"
1092
+ "x-contract-version": "0.5.28"
1093
1093
  }
@@ -471,7 +471,7 @@
471
471
  ],
472
472
  "title": "SeqFlags",
473
473
  "type": "integer",
474
- "x-contract-version": "0.5.27",
474
+ "x-contract-version": "0.5.28",
475
475
  "x-flag-names": {
476
476
  "CHECK": 65536,
477
477
  "INFER": 16,
@@ -722,5 +722,5 @@
722
722
  ],
723
723
  "title": "BenchmarkSpec",
724
724
  "type": "object",
725
- "x-contract-version": "0.5.27"
725
+ "x-contract-version": "0.5.28"
726
726
  }
@@ -40,5 +40,5 @@
40
40
  ],
41
41
  "title": "BenchmarkStatusResponse",
42
42
  "type": "object",
43
- "x-contract-version": "0.5.27"
43
+ "x-contract-version": "0.5.28"
44
44
  }
@@ -79,5 +79,5 @@
79
79
  ],
80
80
  "title": "BootstrapResponse",
81
81
  "type": "object",
82
- "x-contract-version": "0.5.27"
82
+ "x-contract-version": "0.5.28"
83
83
  }
@@ -122,5 +122,5 @@
122
122
  ],
123
123
  "title": "Camera",
124
124
  "type": "object",
125
- "x-contract-version": "0.5.27"
125
+ "x-contract-version": "0.5.28"
126
126
  }
@@ -23,5 +23,5 @@
23
23
  ],
24
24
  "title": "CodeBundle",
25
25
  "type": "object",
26
- "x-contract-version": "0.5.27"
26
+ "x-contract-version": "0.5.28"
27
27
  }
@@ -96,5 +96,5 @@
96
96
  },
97
97
  "title": "Concurrency",
98
98
  "type": "object",
99
- "x-contract-version": "0.5.27"
99
+ "x-contract-version": "0.5.28"
100
100
  }
@@ -55,5 +55,5 @@
55
55
  ],
56
56
  "title": "Conditions",
57
57
  "type": "object",
58
- "x-contract-version": "0.5.27"
58
+ "x-contract-version": "0.5.28"
59
59
  }
@@ -25,5 +25,5 @@
25
25
  ],
26
26
  "title": "ConnectRequest",
27
27
  "type": "object",
28
- "x-contract-version": "0.5.27"
28
+ "x-contract-version": "0.5.28"
29
29
  }
@@ -604,5 +604,5 @@
604
604
  ],
605
605
  "title": "ConnectResponse",
606
606
  "type": "object",
607
- "x-contract-version": "0.5.27"
607
+ "x-contract-version": "0.5.28"
608
608
  }
@@ -18,5 +18,5 @@
18
18
  ],
19
19
  "title": "DeploymentAccepted",
20
20
  "type": "object",
21
- "x-contract-version": "0.5.27"
21
+ "x-contract-version": "0.5.28"
22
22
  }
@@ -833,5 +833,5 @@
833
833
  ],
834
834
  "title": "DeploymentPolicy",
835
835
  "type": "object",
836
- "x-contract-version": "0.5.27"
836
+ "x-contract-version": "0.5.28"
837
837
  }
@@ -1614,5 +1614,5 @@
1614
1614
  ],
1615
1615
  "title": "DeploymentRequest",
1616
1616
  "type": "object",
1617
- "x-contract-version": "0.5.27"
1617
+ "x-contract-version": "0.5.28"
1618
1618
  }
@@ -58,5 +58,5 @@
58
58
  ],
59
59
  "title": "DeploymentStatusResponse",
60
60
  "type": "object",
61
- "x-contract-version": "0.5.27"
61
+ "x-contract-version": "0.5.28"
62
62
  }
@@ -297,5 +297,5 @@
297
297
  ],
298
298
  "title": "Embodiment",
299
299
  "type": "object",
300
- "x-contract-version": "0.5.27"
300
+ "x-contract-version": "0.5.28"
301
301
  }
@@ -94,5 +94,5 @@
94
94
  ],
95
95
  "title": "ErrorBody",
96
96
  "type": "object",
97
- "x-contract-version": "0.5.27"
97
+ "x-contract-version": "0.5.28"
98
98
  }
@@ -22,7 +22,7 @@
22
22
  ],
23
23
  "title": "ErrorCode",
24
24
  "type": "string",
25
- "x-contract-version": "0.5.27",
25
+ "x-contract-version": "0.5.28",
26
26
  "x-errors": {
27
27
  "budget_exhausted": {
28
28
  "exit_code": 6,
@@ -60,5 +60,5 @@
60
60
  ],
61
61
  "title": "Estimate",
62
62
  "type": "object",
63
- "x-contract-version": "0.5.27"
63
+ "x-contract-version": "0.5.28"
64
64
  }
@@ -240,5 +240,5 @@
240
240
  ],
241
241
  "title": "EvalDryRunResponse",
242
242
  "type": "object",
243
- "x-contract-version": "0.5.27"
243
+ "x-contract-version": "0.5.28"
244
244
  }