sequence-base 0.4.29__tar.gz → 0.4.31__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. {sequence_base-0.4.29 → sequence_base-0.4.31}/PKG-INFO +5 -1
  2. {sequence_base-0.4.29 → sequence_base-0.4.31}/pyproject.toml +2 -1
  3. sequence_base-0.4.31/sequence_base/by_name.py +90 -0
  4. sequence_base-0.4.31/sequence_base/codec/marked.py +337 -0
  5. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/constants/version.py +3 -1
  6. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/construct.py +26 -5
  7. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/flags.py +2 -0
  8. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/policy.py +17 -9
  9. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ActRequest.json +1 -1
  10. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ActResponse.json +1 -1
  11. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ActionChunk.json +1 -1
  12. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ActionContract.json +1 -1
  13. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ActionStep.json +1 -1
  14. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Actions.json +1 -1
  15. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/AdapterPair.json +1 -1
  16. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/AddLocal.json +1 -1
  17. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Autoscaler.json +1 -1
  18. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/BenchmarkConditionSpec.json +1 -1
  19. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/BenchmarkRegisterRequest.json +1 -1
  20. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/BenchmarkSpec.json +2 -2
  21. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/BenchmarkStatusResponse.json +1 -1
  22. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/BootstrapResponse.json +1 -1
  23. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Camera.json +1 -1
  24. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/CodeBundle.json +1 -1
  25. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Concurrency.json +1 -1
  26. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Conditions.json +1 -1
  27. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ConnectRequest.json +1 -1
  28. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ConnectResponse.json +1 -1
  29. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/DeploymentAccepted.json +1 -1
  30. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/DeploymentPolicy.json +1 -1
  31. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/DeploymentRequest.json +1 -1
  32. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/DeploymentStatusResponse.json +1 -1
  33. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Embodiment.json +1 -1
  34. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ErrorBody.json +1 -1
  35. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ErrorCodes.json +1 -1
  36. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Estimate.json +1 -1
  37. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/EvalDryRunResponse.json +1 -1
  38. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/EvalReport.json +1 -1
  39. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/EvalRequest.json +1 -1
  40. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/EvalResponse.json +1 -1
  41. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ImageFrame.json +1 -1
  42. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ImageSpec.json +1 -1
  43. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepAddLocal.json +1 -1
  44. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepApt.json +1 -1
  45. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepEnv.json +1 -1
  46. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepProject.json +1 -1
  47. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepRun.json +1 -1
  48. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepTemplateFiles.json +1 -1
  49. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ImageStepUvPip.json +1 -1
  50. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/InferParam.json +1 -1
  51. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Lease.json +1 -1
  52. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/MetricAgg.json +1 -1
  53. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Observation.json +1 -1
  54. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ObservationContract.json +1 -1
  55. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Observations.json +1 -1
  56. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ObserveRequest.json +1 -1
  57. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ObserveResponse.json +1 -1
  58. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/PolicyRegistration.json +7 -6
  59. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/PolicySpec.json +1 -1
  60. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Project.json +1 -1
  61. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Proprioception.json +1 -1
  62. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/RenderSpec.json +1 -1
  63. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/ReplaySpec.json +1 -1
  64. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Resources.json +1 -1
  65. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/RolloutResult.json +1 -1
  66. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/SecretCreateRequest.json +1 -1
  67. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/SecretInfo.json +1 -1
  68. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/SecretRef.json +1 -1
  69. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/SeqFlags.json +1 -1
  70. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Step.json +1 -1
  71. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Stream.json +1 -1
  72. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/TaskAgg.json +1 -1
  73. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/TemplateFiles.json +1 -1
  74. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Tensors.json +1 -1
  75. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Timeouts.json +1 -1
  76. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/UploadTarget.json +1 -1
  77. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/Variant.json +1 -1
  78. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/VolumeFile.json +1 -1
  79. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/VolumeInfo.json +1 -1
  80. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/VolumeMount.json +1 -1
  81. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/VolumeRef.json +1 -1
  82. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/VolumeUploadRequest.json +1 -1
  83. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/VolumeUploadResponse.json +1 -1
  84. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schemas/WarmingError.json +1 -1
  85. {sequence_base-0.4.29 → sequence_base-0.4.31}/.gitignore +0 -0
  86. {sequence_base-0.4.29 → sequence_base-0.4.31}/README.md +0 -0
  87. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/__init__.py +0 -0
  88. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/auth.py +0 -0
  89. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/codec/__init__.py +0 -0
  90. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/codec/image.py +0 -0
  91. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/constants/__init__.py +0 -0
  92. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/constants/cards.py +0 -0
  93. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/constants/headers.py +0 -0
  94. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/constants/limits.py +0 -0
  95. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/__init__.py +0 -0
  96. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/accounts.py +0 -0
  97. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/act.py +0 -0
  98. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/adapters.py +0 -0
  99. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/benchmarks.py +0 -0
  100. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/connect.py +0 -0
  101. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/deployments.py +0 -0
  102. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/errors.py +0 -0
  103. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/eval.py +0 -0
  104. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/io.py +0 -0
  105. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/obs_schedule.py +0 -0
  106. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/observe.py +0 -0
  107. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/own_io.py +0 -0
  108. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/templates.py +0 -0
  109. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/volumes.py +0 -0
  110. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/contract/wire.py +0 -0
  111. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/episode.py +0 -0
  112. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/errors/__init__.py +0 -0
  113. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/errors/transport.py +0 -0
  114. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/feedback.py +0 -0
  115. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/msgpack_io.py +0 -0
  116. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/obs/__init__.py +0 -0
  117. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/obs/log.py +0 -0
  118. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/obs/rid.py +0 -0
  119. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/presets/__init__.py +0 -0
  120. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/presets/frames.py +0 -0
  121. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/presets/robot_arm.py +0 -0
  122. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/schema_export.py +0 -0
  123. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/telemetry/__init__.py +0 -0
  124. {sequence_base-0.4.29 → sequence_base-0.4.31}/sequence_base/telemetry/events.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: sequence-base
3
- Version: 0.4.29
3
+ Version: 0.4.31
4
4
  Summary: Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness.
5
5
  Project-URL: Changelog, https://generalsequences.com/docs/changelog/
6
6
  Requires-Python: >=3.10
@@ -8,6 +8,10 @@ Requires-Dist: pydantic>=2.0
8
8
  Provides-Extra: codec
9
9
  Requires-Dist: numpy>=1.26; extra == 'codec'
10
10
  Requires-Dist: pillow>=10; extra == 'codec'
11
+ Provides-Extra: observe
12
+ Requires-Dist: av>=17; extra == 'observe'
13
+ Requires-Dist: numpy>=1.26; extra == 'observe'
14
+ Requires-Dist: zstandard>=0.22; extra == 'observe'
11
15
  Provides-Extra: test
12
16
  Requires-Dist: hypothesis>=6; extra == 'test'
13
17
  Requires-Dist: jsonschema>=4; extra == 'test'
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "sequence-base"
3
- version = "0.4.29"
3
+ version = "0.4.31"
4
4
  description = "Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness."
5
5
  requires-python = ">=3.10"
6
6
  dependencies = ["pydantic>=2.0"]
@@ -10,6 +10,7 @@ Changelog = "https://generalsequences.com/docs/changelog/"
10
10
 
11
11
  [project.optional-dependencies]
12
12
  codec = ["pillow>=10", "numpy>=1.26"]
13
+ observe = ["numpy>=1.26", "av>=17", "zstandard>=0.22"]
13
14
  test = ["pytest", "jsonschema>=4", "hypothesis>=6", "pyyaml"]
14
15
 
15
16
  [build-system]
@@ -0,0 +1,90 @@
1
+ """
2
+ Which arguments of a method a policy class names (infer=, observe=) an act's inputs fill: the inputs whole, its one
3
+ argument - or, when it takes two or more without a default, each key of the inputs to its argument of that name.
4
+
5
+ In one sentence: `takes_by_name(parameters, platform)` reads the method's arguments - less *args, **kwargs and the ones
6
+ the platform hands it itself (`platform`: INFER_PLATFORM, OBSERVE_PLATFORM) → two or more of them without a default:
7
+ it takes its inputs by name (ByName), each of its arguments a key - one without a default a key the inputs must hold,
8
+ one with a default a key they may → `ByName.call(inputs, what)` makes what it is called with, refusing inputs that are
9
+ no dict, lack a key it needs, or hold one it takes no argument for (unless it takes **kwargs), each named; fewer than
10
+ two: None, the inputs go to its first argument whole and its others with a default are inference parameters, as before.
11
+
12
+ WHY by the arguments without a default: a method that needs two or more could never be handed an act's inputs whole -
13
+ the platform refused to deploy one - so binding them by name changes nothing a class deployed today does, and a method
14
+ that takes one beside its inference parameters keeps taking its inputs whole. The SDK reads the rule when a class is
15
+ declared (seq/named.py, registry.infer_params), the worker when it calls the method (harness/handler/loader.py): one
16
+ rule on both sides.
17
+ """
18
+ from __future__ import annotations
19
+
20
+ import inspect
21
+ from collections.abc import Iterable, Mapping
22
+ from dataclasses import dataclass
23
+ from typing import Any
24
+
25
+ __all__ = ["INFER_PLATFORM", "OBSERVE_PLATFORM", "ByName", "takes_by_name"]
26
+
27
+
28
+
29
+ INFER_PLATFORM = frozenset({"seed", "episode_id", "episode", "feedback", "plan", "robot", "embodiment"})
30
+ OBSERVE_PLATFORM = frozenset({"step"})
31
+
32
+ _VARIADIC = (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD)
33
+
34
+
35
+ @dataclass(frozen=True)
36
+ class ByName:
37
+ """A method that takes an act's inputs by name: `names`, the arguments they fill, in order; `required`, those with
38
+ no default; `positional`, those it takes by position only (before a `/`); `rest`, whether it takes **kwargs - any
39
+ other key of the inputs going there; `platform`, the names the platform hands it itself."""
40
+
41
+ names: tuple[str, ...]
42
+ required: tuple[str, ...]
43
+ positional: tuple[str, ...]
44
+ rest: bool
45
+ platform: frozenset[str]
46
+
47
+ def call(self, inputs: Any, what: str) -> tuple[list, dict]:
48
+ """(args, kwargs) the method `what` ("Model.predict") is called with for an act's `inputs`; ValueError, in its
49
+ words, for inputs that are not a dict of its arguments."""
50
+ takes = ", ".join(n if n in self.required else f"{n} (optional)" for n in self.names)
51
+ if not isinstance(inputs, Mapping):
52
+ raise ValueError(f"{what} takes its inputs by name ({takes}): an act's inputs are a dict of them, and "
53
+ f"these are a {type(inputs).__name__}")
54
+ missing = [n for n in self.required if n not in inputs]
55
+ extra = [k for k in inputs if k not in self.names
56
+ and not (self.rest and isinstance(k, str) and k not in self.platform)]
57
+ if missing or extra:
58
+ held = f"lack {_keys(missing)}" if missing else ""
59
+ if extra:
60
+ held += (" and " if held else "") + f"hold {_keys(extra)}, which it takes no argument for"
61
+ ours = [k for k in extra if k in self.platform]
62
+ mine = f" ({_keys(ours)}: the platform's to hand it)" if ours else ""
63
+ raise ValueError(f"{what} takes its inputs by name ({takes}), and these {held}{mine}")
64
+ args: list = []
65
+ for i, name in enumerate(self.positional):
66
+ if name not in inputs:
67
+ later = next((n for n in self.positional[i + 1:] if n in inputs), None)
68
+ if later is not None:
69
+ raise ValueError(f"{what} takes {name} and {later} by position only, in that order: these hold "
70
+ f"{later!r} and no {name!r}")
71
+ break
72
+ args.append(inputs[name])
73
+ return args, {k: v for k, v in inputs.items() if k not in self.positional}
74
+
75
+
76
+ def takes_by_name(parameters: Iterable[inspect.Parameter], platform: frozenset[str] = frozenset()) -> ByName | None:
77
+ """How a method of these `parameters` (self left out) takes an act's inputs: by name (ByName) when two or more of
78
+ its arguments - *args, **kwargs and `platform`'s left out - have no default, else None: whole, as one argument."""
79
+ every = list(parameters)
80
+ params = [p for p in every if p.kind not in _VARIADIC and p.name not in platform]
81
+ required = tuple(p.name for p in params if p.default is inspect.Parameter.empty)
82
+ if len(required) < 2:
83
+ return None
84
+ return ByName(names=tuple(p.name for p in params), required=required,
85
+ positional=tuple(p.name for p in params if p.kind is inspect.Parameter.POSITIONAL_ONLY),
86
+ rest=any(p.kind is inspect.Parameter.VAR_KEYWORD for p in every), platform=frozenset(platform))
87
+
88
+
89
+ def _keys(names: list) -> str:
90
+ return ", ".join(repr(n) for n in names)
@@ -0,0 +1,337 @@
1
+ """
2
+ Marked observation items: what the observation format puts on the wire. A robot marks items of its own observe data as
3
+ sequence_ai.Video, Depth or Tactile; each marked item travels as the map below and the worker turns it back into the
4
+ array it was, so the policy's observe function reads the data the robot built, item for item.
5
+
6
+
7
+
8
+ In one sentence: the robot's SDK keeps one Encoder per marked item of an episode (its path in the observe data) → each
9
+ observe() puts the map that item's encoder writes in place of the marked value → the worker keeps one Decoder per path
10
+ of each episode (Streams) and puts each map's array back (decode) before the policy's observe function is called.
11
+
12
+ The three kinds, as the user named them:
13
+ - video - an RGB camera frame, HxWx3 uint8: H.264 for low latency - no B-frames, no lookahead, every frame out of the
14
+ encoder as it goes in - with a keyframe every second (KEYFRAME_S) and whenever the robot asks for one (force_keyframe:
15
+ after a reconnect, or after frames it dropped). Lossy (crf VIDEO_CRF).
16
+ - depth - a depth map of any integer or float dtype, in the robot's own unit: lossless. Each frame on its own, every
17
+ value predicted from its left neighbour (an integer's difference, a float's bits XORed) and the bytes zstd-compressed.
18
+ 640x480 sensor depth comes to about 2.3-2.5 times smaller in about a millisecond each way, as small as anything else
19
+ measured and the fastest; frames are not predicted from the frame before, because a sensor's noise is new in every
20
+ frame and that gained nothing.
21
+ - tactile - a tactile reading. Image-like (HxWx3 uint8, what a camera-based sensor gives) is H.264 at high quality
22
+ (crf TACTILE_CRF); any other array, a pressure grid, travels as it is. Named tactile whatever it travels as.
23
+
24
+ The map, version 1 (inside the observe body's msgpack; `data` is bin):
25
+ {"__seq__": "video" | "tactile", "v": 1, "codec": "h264", "w", "h", "n", "kf", "data"}
26
+ w, h: the frame's size (an odd one is padded to even for the encoder and cut back after); n: the frame's number in
27
+ its stream, from 0, never reused - a gap in n is frames the robot dropped; kf: a keyframe, which a decoder can
28
+ start from; data: one H.264 access unit (Annex B, its parameter sets in front of every keyframe).
29
+ {"__seq__": "depth", "v": 1, "codec": "zstd-delta" | "zstd-xor", "dtype", "shape", "data"}
30
+ dtype: numpy's string for it ("<u2", "<f4", ...); shape: the array's; data: the predicted values' bytes, byte
31
+ planes apart, zstd-compressed.
32
+ {"__seq__": "tactile", "v": 1, "codec": "raw", "value"}
33
+ value: the array (or whatever was marked), as msgpack carries any array.
34
+
35
+ WHY a map with a reserved key rather than a msgpack extension type: sequence_base.msgpack_io refuses every extension
36
+ type, so that reading a body runs nothing its sender chose; a map reads as data everywhere.
37
+ WHY the codecs live here: the SDK encodes with the same module the worker decodes with, so the two cannot drift, and one
38
+ round-trip test pins the format (tests/test_marked.py), as codec/image.py does for the frames of the robot-arm preset.
39
+
40
+ Needs numpy, PyAV (av) and zstandard - `pip install 'sequence-base[observe]'`; each is imported where a codec first
41
+ needs it, so nothing here is needed by a robot that marks nothing.
42
+
43
+ enc = encoder("video"); wire = enc.encode(frame); enc.force_keyframe() # the robot, one per marked item
44
+ streams = Streams(); value = decode(value, streams) # the worker, one Streams per episode
45
+ """
46
+ from __future__ import annotations
47
+
48
+ import time
49
+ from fractions import Fraction
50
+ from typing import Any, Callable
51
+
52
+ __all__ = ["KINDS", "MARK", "VERSION", "Decoder", "Encoder", "MarkedError", "NeedsKeyframe", "Streams", "decode",
53
+ "decode_one", "encoder", "is_marked"]
54
+
55
+ MARK = "__seq__"
56
+ VERSION = 1
57
+ KINDS = ("video", "depth", "tactile")
58
+
59
+ KEYFRAME_S = 1.0
60
+ VIDEO_CRF = 23
61
+ TACTILE_CRF = 12
62
+ _PRESET = "ultrafast"
63
+ _ZSTD_LEVEL = 1
64
+
65
+
66
+ class MarkedError(ValueError):
67
+ """A value a marker cannot carry (encoding), or a map that is not one this reads (decoding)."""
68
+
69
+
70
+ class NeedsKeyframe(MarkedError):
71
+ """A video frame that follows a gap in its stream and is not a keyframe: nothing before it to decode it from. The
72
+ robot's encoder sends a keyframe after any frame it dropped (force_keyframe), so this is never the robot's data."""
73
+
74
+
75
+ def is_marked(value: Any) -> bool:
76
+ return isinstance(value, dict) and value.get(MARK) in KINDS
77
+
78
+
79
+ def _wire(kind: str, **fields: Any) -> dict:
80
+ """A marked item's map: its kind under MARK, the version, then its codec's fields."""
81
+ return dict([(MARK, kind), ("v", VERSION), *fields.items()])
82
+
83
+
84
+
85
+ class Encoder:
86
+ """One marked item's stream: `encode(value)` -> its map for this observe; `force_keyframe()` makes the next video
87
+ frame a keyframe."""
88
+
89
+ kind = ""
90
+
91
+ def encode(self, value: Any) -> dict:
92
+ raise NotImplementedError
93
+
94
+ def force_keyframe(self) -> None:
95
+ """Nothing to do for a kind with no state between frames."""
96
+
97
+
98
+ def encoder(kind: str, *, clock: Callable[[], float] = time.monotonic) -> Encoder:
99
+ """A new stream's encoder for `kind` ("video", "depth" or "tactile")."""
100
+ if kind == "video":
101
+ return _VideoEncoder(clock)
102
+ if kind == "depth":
103
+ return _DepthEncoder()
104
+ if kind == "tactile":
105
+ return _TactileEncoder(clock)
106
+ raise MarkedError(f"no marker {kind!r}: the kinds are {', '.join(KINDS)}")
107
+
108
+
109
+ class _H264:
110
+ """libx264 for one stream: low latency, every frame out at once, a forced keyframe an IDR the decoder starts from."""
111
+
112
+ def __init__(self, crf: int, clock: Callable[[], float]) -> None:
113
+ self._crf, self._clock = crf, clock
114
+ self._ctx = None
115
+ self._size: tuple[int, int] | None = None
116
+ self._n = 0
117
+ self._key_at: float | None = None
118
+ self._force = True
119
+
120
+ def force_keyframe(self) -> None:
121
+ self._force = True
122
+
123
+ def encode(self, frame: Any, kind: str) -> dict:
124
+ import av
125
+ import numpy as np
126
+
127
+ a = np.asarray(frame)
128
+ if a.dtype != np.uint8 or a.ndim != 3 or a.shape[2] != 3 or a.shape[0] < 2 or a.shape[1] < 2:
129
+ raise MarkedError(f"a {kind} frame is an RGB image, HxWx3 uint8; got {a.dtype} {tuple(a.shape)} - send other "
130
+ "arrays unmarked, or as Depth or Tactile")
131
+ h, w = int(a.shape[0]), int(a.shape[1])
132
+ if (h, w) != self._size:
133
+ self._open(av, h, w)
134
+ if h % 2 or w % 2:
135
+ a = np.pad(a, ((0, h % 2), (0, w % 2), (0, 0)), mode="edge")
136
+ now = self._clock()
137
+ key = self._force or self._key_at is None or now - self._key_at >= KEYFRAME_S
138
+ vf = av.VideoFrame.from_ndarray(np.ascontiguousarray(a), format="rgb24").reformat(format="yuv420p")
139
+ vf.pts = self._n
140
+ if key:
141
+ vf.pict_type = av.video.frame.PictureType.I
142
+ packets = self._ctx.encode(vf)
143
+ if len(packets) != 1:
144
+ raise MarkedError(f"the H.264 encoder answered {len(packets)} packets for one frame")
145
+ data = bytes(packets[0])
146
+ kf = bool(packets[0].is_keyframe)
147
+ if kf:
148
+ self._key_at, self._force = now, False
149
+ n, self._n = self._n, self._n + 1
150
+ return _wire(kind, codec="h264", w=w, h=h, n=n, kf=kf, data=data)
151
+
152
+ def _open(self, av, h: int, w: int) -> None:
153
+ ctx = av.CodecContext.create("libx264", "w")
154
+ ctx.width, ctx.height, ctx.pix_fmt = w + w % 2, h + h % 2, "yuv420p"
155
+ ctx.time_base = Fraction(1, 30)
156
+
157
+ ctx.options = {"preset": _PRESET, "tune": "zerolatency", "crf": str(self._crf), "forced-idr": "1",
158
+ "x264-params": "keyint=1000000:min-keyint=1:scenecut=0:bframes=0"}
159
+ ctx.thread_count = 1
160
+ ctx.open()
161
+ self._ctx, self._size, self._force = ctx, (h, w), True
162
+
163
+
164
+ class _VideoEncoder(Encoder):
165
+ kind = "video"
166
+
167
+ def __init__(self, clock: Callable[[], float]) -> None:
168
+ self._h264 = _H264(VIDEO_CRF, clock)
169
+
170
+ def encode(self, value: Any) -> dict:
171
+ return self._h264.encode(value, "video")
172
+
173
+ def force_keyframe(self) -> None:
174
+ self._h264.force_keyframe()
175
+
176
+
177
+ class _DepthEncoder(Encoder):
178
+ kind = "depth"
179
+
180
+ def encode(self, value: Any) -> dict:
181
+ import numpy as np
182
+
183
+ a = np.asarray(value)
184
+ if a.dtype.kind not in "uif" or a.ndim < 1 or a.size == 0:
185
+ raise MarkedError(f"a depth map is a non-empty array of integers or floats; got {a.dtype} {tuple(a.shape)}")
186
+ codec, data = _predict(a)
187
+ return _wire("depth", codec=codec, dtype=a.dtype.str, shape=list(a.shape), data=data)
188
+
189
+
190
+ class _TactileEncoder(Encoder):
191
+ kind = "tactile"
192
+
193
+ def __init__(self, clock: Callable[[], float]) -> None:
194
+ self._h264 = _H264(TACTILE_CRF, clock)
195
+
196
+ def encode(self, value: Any) -> dict:
197
+ import numpy as np
198
+
199
+ a = np.asarray(value) if not isinstance(value, (bytes, str, dict)) else None
200
+ if a is not None and a.dtype == np.uint8 and a.ndim == 3 and a.shape[2] == 3 and min(a.shape[:2]) >= 2:
201
+ return self._h264.encode(a, "tactile")
202
+ return _wire("tactile", codec="raw", value=value)
203
+
204
+ def force_keyframe(self) -> None:
205
+ self._h264.force_keyframe()
206
+
207
+
208
+
209
+ def _predict(a) -> tuple[str, bytes]:
210
+ import numpy as np
211
+ import zstandard
212
+
213
+ le = a.astype(a.dtype.newbyteorder("<"), copy=False)
214
+ size = le.dtype.itemsize
215
+ rows = np.ascontiguousarray(le).reshape(-1, le.shape[-1]).view(f"<u{size}")
216
+ out = rows.copy()
217
+ if le.dtype.kind == "f":
218
+ out[:, 1:] = rows[:, 1:] ^ rows[:, :-1]
219
+ codec = "zstd-xor"
220
+ else:
221
+ out[:, 1:] = rows[:, 1:] - rows[:, :-1]
222
+ signed = out.view(f"<i{size}")
223
+ out = ((signed << 1) ^ (signed >> (8 * size - 1))).view(f"<u{size}")
224
+ codec = "zstd-delta"
225
+ planes = out.view(np.uint8).reshape(-1, size).T.tobytes()
226
+ return codec, zstandard.ZstdCompressor(level=_ZSTD_LEVEL).compress(planes)
227
+
228
+
229
+ def _unpredict(m: dict):
230
+ import numpy as np
231
+ import zstandard
232
+
233
+ dtype, shape, codec = np.dtype(m["dtype"]), tuple(int(s) for s in m["shape"]), m["codec"]
234
+ size = dtype.itemsize
235
+ cols = shape[-1]
236
+ raw = zstandard.ZstdDecompressor().decompress(m["data"])
237
+ count = int(np.prod(shape))
238
+ if len(raw) != count * size or codec not in ("zstd-delta", "zstd-xor"):
239
+ raise MarkedError(f"a depth map that does not read back: {codec} {m['dtype']} {list(shape)}")
240
+ rows = np.frombuffer(raw, np.uint8).reshape(size, -1).T.copy().view(f"<u{size}").reshape(-1, cols)
241
+ if codec == "zstd-xor":
242
+ values = np.bitwise_xor.accumulate(rows, axis=1)
243
+ else:
244
+ signed = rows.view(f"<i{size}")
245
+ steps = ((rows >> 1).view(f"<i{size}") ^ -(signed & 1)).view(f"<u{size}")
246
+ values = np.cumsum(steps, axis=1, dtype=f"<u{size}")
247
+ return values.view(dtype.newbyteorder("<")).reshape(shape).astype(dtype, copy=False)
248
+
249
+
250
+
251
+ class Decoder:
252
+ """One marked item's stream: `decode(map)` -> the array it carries."""
253
+
254
+ def decode(self, m: dict) -> Any:
255
+ return decode_one(m)
256
+
257
+
258
+ class _VideoDecoder(Decoder):
259
+ """H.264 for one stream, frames in order; after a gap in n it waits for a keyframe (NeedsKeyframe until then)."""
260
+
261
+ def __init__(self) -> None:
262
+ self._ctx = None
263
+ self._last: int | None = None
264
+ self._whole = False
265
+
266
+ def decode(self, m: dict) -> Any:
267
+ import av
268
+
269
+ n, kf = int(m["n"]), bool(m["kf"])
270
+ if self._last is not None and n != self._last + 1:
271
+ self._whole = False
272
+ self._last = n
273
+ if kf:
274
+ self._whole = True
275
+ if not self._whole:
276
+ raise NeedsKeyframe(f"{m.get(MARK)} frame {n} follows a gap in its stream and is not a keyframe")
277
+ if self._ctx is None:
278
+ self._ctx = av.CodecContext.create("h264", "r")
279
+ self._ctx.thread_count = 1
280
+ frames = self._ctx.decode(av.Packet(m["data"]))
281
+ if not frames:
282
+ raise MarkedError(f"{m.get(MARK)} frame {n} decoded to nothing")
283
+ import numpy as np
284
+
285
+ rgb = frames[-1].to_ndarray(format="rgb24")
286
+ h, w = int(m["h"]), int(m["w"])
287
+ return rgb if rgb.shape[:2] == (h, w) else np.ascontiguousarray(rgb[:h, :w])
288
+
289
+
290
+ def decode_one(m: dict) -> Any:
291
+ """A map with no state between frames - depth, or tactile as it is - back to its value. Video needs its stream's
292
+ Decoder (Streams)."""
293
+ if not is_marked(m) or int(m.get("v") or 0) != VERSION:
294
+ raise MarkedError(f"not a marked item of version {VERSION}: {sorted(m) if isinstance(m, dict) else type(m)}")
295
+ codec = m.get("codec")
296
+ if m[MARK] == "depth":
297
+ return _unpredict(m)
298
+ if m[MARK] == "tactile" and codec == "raw":
299
+ return m.get("value")
300
+ raise MarkedError(f"a {m[MARK]} item in {codec!r} decodes in its stream's order (Streams)")
301
+
302
+
303
+ class Streams:
304
+ """One episode's decoders, by the path of each marked item in its observe data."""
305
+
306
+ def __init__(self) -> None:
307
+ self._decoders: dict[tuple, Decoder] = {}
308
+
309
+ def decoder(self, path: tuple, m: dict) -> Decoder:
310
+ d = self._decoders.get(path)
311
+ if d is None:
312
+ d = self._decoders[path] = _VideoDecoder() if m.get("codec") == "h264" else Decoder()
313
+ return d
314
+
315
+
316
+ def decode(value: Any, streams: Streams, path: tuple = ()) -> Any:
317
+ """`value` with every marked map in it - in dicts and lists, at any depth - replaced by its array; everything else
318
+ as it was, the same objects. A map that is not marked is walked; one that is decodes with its path's decoder."""
319
+ if isinstance(value, dict):
320
+ if is_marked(value):
321
+ return streams.decoder(path, value).decode(value)
322
+ if not any(_holds_mark(v) for v in value.values()):
323
+ return value
324
+ return {k: decode(v, streams, path + (k,)) for k, v in value.items()}
325
+ if isinstance(value, list):
326
+ if not any(_holds_mark(v) for v in value):
327
+ return value
328
+ return [decode(v, streams, path + (i,)) for i, v in enumerate(value)]
329
+ return value
330
+
331
+
332
+ def _holds_mark(value: Any) -> bool:
333
+ if isinstance(value, dict):
334
+ return is_marked(value) or any(_holds_mark(v) for v in value.values())
335
+ if isinstance(value, list):
336
+ return any(_holds_mark(v) for v in value)
337
+ return False
@@ -4,7 +4,9 @@
4
4
 
5
5
 
6
6
 
7
- CONTRACT_VERSION = "0.5.26"
7
+ CONTRACT_VERSION = "0.5.28"
8
+
9
+
8
10
 
9
11
 
10
12
 
@@ -1,10 +1,12 @@
1
1
  """
2
2
  How a policy class that names its own methods is built (PolicyRegistration.constructor): its constructor - or the
3
- factory it names - called with the arguments its decorator was given, once two forms in them are filled in.
3
+ factory it names, a classmethod of it or a module's function - called with the arguments its decorator was given, once
4
+ two forms in them are filled in.
4
5
 
5
6
  In one sentence: a value {"$config": path, "$set": {...}} becomes the YAML or JSON file at `path` - from the folder of
6
7
  the class's file - read where the policy runs, with `$set`'s keys laid over its top level → in every string,
7
8
  "{weights}" becomes the weights directory and "{weights.<part>}" a named part's → `cls.<make>(*args, **kwargs)`, or
9
+ `function(*args, **kwargs)` for make="package.module:function" - the module imported where the policy runs - or
8
10
  `cls(*args, **kwargs)` without a factory.
9
11
 
10
12
  WHY a config file is read where the policy runs: it is part of the project (XPolicyLab's deploy.yml beside its
@@ -14,12 +16,14 @@ worker's directory.
14
16
  """
15
17
  from __future__ import annotations
16
18
 
19
+ import importlib
17
20
  import json
18
21
  import os
19
22
  import re
20
23
  from typing import Any
21
24
 
22
- __all__ = ["CONFIG_KEY", "SET_KEY", "ConstructError", "build", "config", "expand", "is_config", "read_config"]
25
+ __all__ = ["CONFIG_KEY", "SET_KEY", "ConstructError", "build", "config", "expand", "is_config", "maker",
26
+ "read_config"]
23
27
 
24
28
  CONFIG_KEY, SET_KEY = "$config", "$set"
25
29
  _WEIGHTS = re.compile(r"\{weights(?:\.([A-Za-z_][A-Za-z0-9_-]*))?\}")
@@ -96,8 +100,25 @@ def _weights_of(part: str | None, weights: Any) -> str:
96
100
 
97
101
 
98
102
  def build(cls: type, constructor: Any, *, base_dir: str, weights: Any) -> Any:
99
- """The instance `constructor` (a PolicyConstructor) makes of `cls`: `cls.<make>(*args, **kwargs)` or `cls(...)`."""
103
+ """The instance `constructor` (a PolicyConstructor) makes of `cls`: `cls.<make>(*args, **kwargs)`, the function
104
+ make="package.module:function" names called so, or `cls(...)`."""
100
105
  args = expand(list(constructor.args), base_dir=base_dir, weights=weights)
101
106
  kwargs = expand(dict(constructor.kwargs), base_dir=base_dir, weights=weights)
102
- make = getattr(cls, constructor.make) if constructor.make else cls
103
- return make(*args, **kwargs)
107
+ return maker(cls, constructor.make)(*args, **kwargs)
108
+
109
+
110
+ def maker(cls: type, make: str | None) -> Any:
111
+ """What builds the instance: `cls` for no make=, its classmethod or staticmethod `make`, or - make written
112
+ "package.module:function" - that module's function, the module imported (as the policy's code imports it).
113
+ """
114
+ if not make:
115
+ return cls
116
+ module, colon, name = make.rpartition(":")
117
+ if not colon:
118
+ return getattr(cls, make)
119
+ found = getattr(importlib.import_module(module), name, None)
120
+ if found is None:
121
+ raise ConstructError(f"make={make!r}: {module} has no function {name!r}")
122
+ if not callable(found):
123
+ raise ConstructError(f"make={make!r}: {module}.{name} is not callable ({type(found).__name__})")
124
+ return found
@@ -48,6 +48,8 @@ class SeqFlags(IntFlag):
48
48
 
49
49
 
50
50
 
51
+
52
+
51
53
  SAMPLE = 524288
52
54
 
53
55
 
@@ -482,19 +482,26 @@ def check_weights_parts(weights_name: str | None, extra: dict) -> None:
482
482
  raise ValueError(f"at most {MAX_WEIGHTS_PARTS} weights parts")
483
483
 
484
484
 
485
+
486
+ _MAKE = r"^(?:[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*:)?[A-Za-z_][A-Za-z0-9_]*$"
487
+
488
+
485
489
  class PolicyConstructor(BaseModel):
486
490
  """How the worker builds a policy class that names its own methods (`@seq.policy(infer="get_action", ...)`): its
487
- own constructor, or the factory it names (`make`, e.g. LeRobot's `from_pretrained`), called with `args` and
488
- `kwargs` where `@seq.load` would run - the replica's cold start, the weights on disk. The values are the user's own,
489
- as JSON carries them, but for two forms the worker fills in first (sequence_base.construct.expand): a config file's
490
- contents, `{"$config": <its path from the class's file>, "$set": {<keys laid over it>}}`, YAML or JSON, and in any
491
- string `"{weights}"` (the weights directory) or `"{weights.<part>}"` (a named part's).
491
+ own constructor, or the factory it names (`make`: a classmethod of it, e.g. LeRobot's `from_pretrained`, or a
492
+ module's function, "package.module:function", whose return is the instance its methods are called on), called
493
+ with `args` and `kwargs` where `@seq.load` would run - the replica's cold start, the weights on disk. The values are
494
+ the user's own, as JSON carries them, but for two forms the worker fills in first (sequence_base.construct.expand):
495
+ a config file's contents, `{"$config": <its path from the class's file>, "$set": {<keys laid over it>}}`, YAML or
496
+ JSON, and in any string `"{weights}"` (the weights directory) or `"{weights.<part>}"` (a named part's).
497
+
492
498
  """
493
499
 
494
500
  model_config = ConfigDict(extra="forbid")
495
501
 
496
- make: str | None = Field(default=None, pattern=r"^[A-Za-z_][A-Za-z0-9_]*$",
497
- description="a classmethod or staticmethod that returns the instance; None: the class")
502
+ make: str | None = Field(default=None, max_length=512, pattern=_MAKE,
503
+ description="a classmethod or staticmethod that returns the instance, or a module's "
504
+ "function, 'package.module:function' (contract 0.5.27); None: the class")
498
505
  args: list[Any] = Field(default_factory=list, description="positional arguments, in order")
499
506
  kwargs: dict[str, Any] = Field(default_factory=dict, description="keyword arguments")
500
507
 
@@ -509,8 +516,9 @@ class PolicyRegistration(BaseModel):
509
516
  act carries, before INFER is called with none), RESET (an episode's first act) and ON_EPISODE (its end).
510
517
 
511
518
  Any policy may bind SAMPLE (contract 0.5.24, `@seq.policy(sample="<method>")`): the method returning one episode's
512
- act inputs, in order, each as an act carries it - what its build check acts with and its worker times its infer on
513
- when no contract says what it reads."""
519
+ calls in order, as a robot makes them - ("observe", x) for its observe(x) and ("act",) for its act(), which carries
520
+ nothing, every input going through observe - what its build check plays as the robot's own calls go, and its worker
521
+ times its infer on (the acts) when no contract says what it reads."""
514
522
 
515
523
  model_config = ConfigDict(extra="forbid")
516
524
 
@@ -107,5 +107,5 @@
107
107
  ],
108
108
  "title": "ActRequest",
109
109
  "type": "object",
110
- "x-contract-version": "0.5.26"
110
+ "x-contract-version": "0.5.28"
111
111
  }
@@ -89,5 +89,5 @@
89
89
  ],
90
90
  "title": "ActResponse",
91
91
  "type": "object",
92
- "x-contract-version": "0.5.26"
92
+ "x-contract-version": "0.5.28"
93
93
  }
@@ -99,5 +99,5 @@
99
99
  ],
100
100
  "title": "ActionChunk",
101
101
  "type": "object",
102
- "x-contract-version": "0.5.26"
102
+ "x-contract-version": "0.5.28"
103
103
  }
@@ -96,5 +96,5 @@
96
96
  ],
97
97
  "title": "ActionContract",
98
98
  "type": "object",
99
- "x-contract-version": "0.5.26"
99
+ "x-contract-version": "0.5.28"
100
100
  }
@@ -19,5 +19,5 @@
19
19
  ],
20
20
  "title": "ActionStep",
21
21
  "type": "object",
22
- "x-contract-version": "0.5.26"
22
+ "x-contract-version": "0.5.28"
23
23
  }
@@ -59,5 +59,5 @@
59
59
  },
60
60
  "title": "Actions",
61
61
  "type": "object",
62
- "x-contract-version": "0.5.26"
62
+ "x-contract-version": "0.5.28"
63
63
  }
@@ -17,5 +17,5 @@
17
17
  ],
18
18
  "title": "AdapterPair",
19
19
  "type": "object",
20
- "x-contract-version": "0.5.26"
20
+ "x-contract-version": "0.5.28"
21
21
  }
@@ -23,5 +23,5 @@
23
23
  ],
24
24
  "title": "AddLocal",
25
25
  "type": "object",
26
- "x-contract-version": "0.5.26"
26
+ "x-contract-version": "0.5.28"
27
27
  }