sequence-base 0.4.22__tar.gz → 0.4.24__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sequence_base-0.4.22 → sequence_base-0.4.24}/PKG-INFO +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/pyproject.toml +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/constants/version.py +3 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/contract/deployments.py +13 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/contract/policy.py +14 -0
- sequence_base-0.4.24/sequence_base/msgpack_io.py +420 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ActRequest.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ActResponse.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ActionChunk.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ActionContract.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ActionStep.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/AdapterPair.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/AddLocal.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/Autoscaler.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/BenchmarkConditionSpec.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/BenchmarkRegisterRequest.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/BenchmarkSpec.json +2 -2
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/BenchmarkStatusResponse.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/BootstrapResponse.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/Camera.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/CodeBundle.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/Concurrency.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/Conditions.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ConnectRequest.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ConnectResponse.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/DeploymentAccepted.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/DeploymentPolicy.json +33 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/DeploymentRequest.json +33 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/DeploymentStatusResponse.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/Embodiment.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ErrorBody.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ErrorCodes.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/Estimate.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/EvalDryRunResponse.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/EvalReport.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/EvalRequest.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/EvalResponse.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ImageFrame.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ImageSpec.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ImageStepAddLocal.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ImageStepApt.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ImageStepEnv.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ImageStepRun.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ImageStepTemplateFiles.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ImageStepUvPip.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/InferParam.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/Lease.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/MetricAgg.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/Observation.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ObservationContract.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/PolicyRegistration.json +34 -2
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/PolicySpec.json +33 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/Proprioception.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/RenderSpec.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/ReplaySpec.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/Resources.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/RolloutResult.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/SecretCreateRequest.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/SecretInfo.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/SecretRef.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/SeqFlags.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/Step.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/TaskAgg.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/TemplateFiles.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/Timeouts.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/UploadTarget.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/Variant.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/VolumeFile.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/VolumeInfo.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/VolumeMount.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/VolumeRef.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/VolumeUploadRequest.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/VolumeUploadResponse.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schemas/WarmingError.json +1 -1
- {sequence_base-0.4.22 → sequence_base-0.4.24}/.gitignore +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/README.md +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/__init__.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/auth.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/codec/__init__.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/codec/image.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/constants/__init__.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/constants/cards.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/constants/headers.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/constants/limits.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/contract/__init__.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/contract/accounts.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/contract/act.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/contract/adapters.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/contract/benchmarks.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/contract/connect.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/contract/errors.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/contract/eval.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/contract/flags.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/contract/io.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/contract/templates.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/contract/volumes.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/contract/wire.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/errors/__init__.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/errors/transport.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/obs/__init__.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/obs/log.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/obs/rid.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/presets/__init__.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/presets/frames.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/presets/robot_arm.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/schema_export.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/telemetry/__init__.py +0 -0
- {sequence_base-0.4.22 → sequence_base-0.4.24}/sequence_base/telemetry/events.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sequence-base
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.24
|
|
4
4
|
Summary: Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness.
|
|
5
5
|
Project-URL: Changelog, https://generalsequences.com/docs/changelog/
|
|
6
6
|
Requires-Python: >=3.10
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sequence-base"
|
|
3
|
-
version = "0.4.
|
|
3
|
+
version = "0.4.24"
|
|
4
4
|
description = "Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness."
|
|
5
5
|
requires-python = ">=3.10"
|
|
6
6
|
dependencies = ["pydantic>=2.0"]
|
|
@@ -202,6 +202,19 @@ class DeploymentPolicy(BaseModel):
|
|
|
202
202
|
|
|
203
203
|
plan_every_s: float | None = Field(default=None, gt=0)
|
|
204
204
|
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
own_observations: Literal["latest", "since_last_act"] | None = None
|
|
209
|
+
own_actions: Literal["one", "steps"] | None = None
|
|
210
|
+
|
|
211
|
+
@model_validator(mode="after")
|
|
212
|
+
def _own_io(self) -> "DeploymentPolicy":
|
|
213
|
+
if (self.own_observations or self.own_actions) and (self.action or self.observation or self.embodiments):
|
|
214
|
+
raise ValueError("own_observations and own_actions are for a policy of its own inputs and outputs; this one "
|
|
215
|
+
"declares the robot-arm contract, where observation history and the action chunk say it")
|
|
216
|
+
return self
|
|
217
|
+
|
|
205
218
|
@model_validator(mode="after")
|
|
206
219
|
def _param_names(self) -> "DeploymentPolicy":
|
|
207
220
|
for name in self.params or {}:
|
|
@@ -281,6 +281,13 @@ class PolicySpec(BaseModel):
|
|
|
281
281
|
|
|
282
282
|
family: str | None = Field(default=None, pattern=KIND_NAME)
|
|
283
283
|
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
own_observations: Literal["latest", "since_last_act"] | None = None
|
|
289
|
+
own_actions: Literal["one", "steps"] | None = None
|
|
290
|
+
|
|
284
291
|
@field_validator("weights")
|
|
285
292
|
@classmethod
|
|
286
293
|
def _weights_scheme(cls, v: str | None) -> str | None:
|
|
@@ -295,6 +302,13 @@ class PolicySpec(BaseModel):
|
|
|
295
302
|
check_weights_uri(uri, f"weights[{name!r}]")
|
|
296
303
|
return self
|
|
297
304
|
|
|
305
|
+
@model_validator(mode="after")
|
|
306
|
+
def _own_io(self) -> "PolicySpec":
|
|
307
|
+
if (self.own_observations or self.own_actions) and (self.action or self.observation or self.embodiments):
|
|
308
|
+
raise ValueError("own_observations and own_actions are for a policy of its own inputs and outputs; this one "
|
|
309
|
+
"declares the robot-arm contract, where observation history and the action chunk say it")
|
|
310
|
+
return self
|
|
311
|
+
|
|
298
312
|
|
|
299
313
|
|
|
300
314
|
WEIGHTS_PART_NAME = re.compile(r"[a-z][a-z0-9_-]{0,31}")
|
|
@@ -0,0 +1,420 @@
|
|
|
1
|
+
"""
|
|
2
|
+
An act as msgpack (application/x-seq-msgpack, version 1): an own-I/O envelope with its bytes as bytes and its numpy
|
|
3
|
+
arrays as their raw data, where JSON carries bytes as base64 - a third more on the wire - and arrays as lists of numbers.
|
|
4
|
+
Written in plain Python, so neither a worker nor a client installs a msgpack package for it.
|
|
5
|
+
|
|
6
|
+
In one sentence: a client takes the act's envelope ({"inputs", "seed", "episode_id", "params", ...}) → `pack` writes it
|
|
7
|
+
as msgpack, each numpy array as the map msgpack-numpy writes ({"nd": True, "type", "kind", "shape", "data"}) → a worker
|
|
8
|
+
that names FORMAT among its formats `unpack`s it back to the same dict, arrays as arrays, and serves it as it serves a
|
|
9
|
+
JSON act; an answer asked for in msgpack (Accept: MEDIA_TYPE) comes back the same way - its arrays, to a client
|
|
10
|
+
without numpy, as the lists their tolist() gives, what the answer in JSON holds.
|
|
11
|
+
|
|
12
|
+
WHY a media type of its own: a worker reads a body that is not JSON as the policy's own inputs and hands it on
|
|
13
|
+
untouched - an `application/msgpack` body included, which some policies decode themselves - so a client sends this
|
|
14
|
+
type only to a worker that says it reads it.
|
|
15
|
+
WHY both array layouts are read: the two ecosystems that send numpy over msgpack write arrays differently -
|
|
16
|
+
msgpack-numpy as {"nd", "type", "kind", "shape", "data"}, openpi as {"__ndarray__", "data", "dtype", "shape"} - and a
|
|
17
|
+
policy's own client may use either. {XPolicyLab 5d0be14 client_server/ws/protocol/codec.py "msgpack_numpy.encode";
|
|
18
|
+
openpi-client msgpack_numpy.py "{b\"__ndarray__\": True, b\"data\": obj.tobytes(), b\"dtype\": obj.dtype.str"}
|
|
19
|
+
WHY no array that holds objects: msgpack-numpy writes one with pickle, and reading that back runs code the sender chose;
|
|
20
|
+
a dtype that holds objects, a structured one and every msgpack extension type are refused, so nothing read here runs
|
|
21
|
+
anything.
|
|
22
|
+
|
|
23
|
+
body = msgpack_io.pack(envelope) # MsgpackError: something msgpack cannot carry
|
|
24
|
+
envelope = msgpack_io.unpack(body) # MsgpackError: not one msgpack value, or one this refuses
|
|
25
|
+
"""
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import re
|
|
29
|
+
import struct
|
|
30
|
+
import sys
|
|
31
|
+
from typing import Any, NamedTuple
|
|
32
|
+
|
|
33
|
+
__all__ = ["FORMAT", "MAX_DEPTH", "MEDIA_TYPE", "MsgpackError", "pack", "unpack"]
|
|
34
|
+
|
|
35
|
+
MEDIA_TYPE = "application/x-seq-msgpack"
|
|
36
|
+
|
|
37
|
+
FORMAT = "x-seq-msgpack/1"
|
|
38
|
+
|
|
39
|
+
MAX_DEPTH = 64
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class MsgpackError(ValueError):
|
|
43
|
+
"""A value msgpack cannot carry (packing), or a body that is not exactly one msgpack value this reads (unpacking)."""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def pack(obj: Any) -> bytes:
|
|
48
|
+
"""`obj` as msgpack: None, bools, ints (-2**63 to 2**64 - 1), floats, str, bytes, lists and tuples, dicts (keys
|
|
49
|
+
str, bytes or int), numpy arrays and scalars - as msgpack-numpy writes them - inside any of those."""
|
|
50
|
+
out: list[bytes] = []
|
|
51
|
+
_pack(obj, out, 0)
|
|
52
|
+
return b"".join(out)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _pack(obj: Any, out: list, depth: int) -> None:
|
|
56
|
+
if depth > MAX_DEPTH:
|
|
57
|
+
raise MsgpackError(f"nested deeper than {MAX_DEPTH} levels")
|
|
58
|
+
if obj is None:
|
|
59
|
+
out.append(b"\xc0")
|
|
60
|
+
elif obj is True:
|
|
61
|
+
out.append(b"\xc3")
|
|
62
|
+
elif obj is False:
|
|
63
|
+
out.append(b"\xc2")
|
|
64
|
+
elif isinstance(obj, int):
|
|
65
|
+
_pack_int(obj, out)
|
|
66
|
+
elif isinstance(obj, float):
|
|
67
|
+
out.append(struct.pack(">Bd", 0xCB, obj))
|
|
68
|
+
elif isinstance(obj, str):
|
|
69
|
+
try:
|
|
70
|
+
raw = obj.encode("utf-8")
|
|
71
|
+
except UnicodeEncodeError as exc:
|
|
72
|
+
raise MsgpackError(f"a string that is not UTF-8: {exc.reason}") from None
|
|
73
|
+
_pack_head(len(raw), out, fix=(0xA0, 31), sizes=((0xD9, ">BB", 0xFF), (0xDA, ">BH", 0xFFFF),
|
|
74
|
+
(0xDB, ">BI", 0xFFFFFFFF)), what="string")
|
|
75
|
+
out.append(raw)
|
|
76
|
+
elif isinstance(obj, (bytes, bytearray, memoryview)):
|
|
77
|
+
raw = bytes(obj)
|
|
78
|
+
_pack_head(len(raw), out, fix=None, sizes=((0xC4, ">BB", 0xFF), (0xC5, ">BH", 0xFFFF),
|
|
79
|
+
(0xC6, ">BI", 0xFFFFFFFF)), what="bytes")
|
|
80
|
+
out.append(raw)
|
|
81
|
+
elif isinstance(obj, (list, tuple)):
|
|
82
|
+
_pack_head(len(obj), out, fix=(0x90, 15), sizes=((0xDC, ">BH", 0xFFFF), (0xDD, ">BI", 0xFFFFFFFF)),
|
|
83
|
+
what="list")
|
|
84
|
+
for item in obj:
|
|
85
|
+
_pack(item, out, depth + 1)
|
|
86
|
+
elif isinstance(obj, dict):
|
|
87
|
+
_pack_head(len(obj), out, fix=(0x80, 15), sizes=((0xDE, ">BH", 0xFFFF), (0xDF, ">BI", 0xFFFFFFFF)),
|
|
88
|
+
what="dict")
|
|
89
|
+
for key, value in obj.items():
|
|
90
|
+
if not isinstance(key, (str, bytes, int)) or isinstance(key, bool):
|
|
91
|
+
raise MsgpackError(f"a dict key that is not str, bytes or int: {type(key).__name__}")
|
|
92
|
+
_pack(key, out, depth + 1)
|
|
93
|
+
_pack(value, out, depth + 1)
|
|
94
|
+
else:
|
|
95
|
+
arr = _numpy_map(obj)
|
|
96
|
+
if arr is None:
|
|
97
|
+
raise MsgpackError(f"{type(obj).__name__} cannot be sent as msgpack (JSON values, bytes and numpy arrays "
|
|
98
|
+
"can)")
|
|
99
|
+
_pack(arr, out, depth)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _pack_int(n: int, out: list) -> None:
|
|
103
|
+
if 0 <= n <= 0x7F:
|
|
104
|
+
out.append(bytes((n,)))
|
|
105
|
+
elif -32 <= n < 0:
|
|
106
|
+
out.append(struct.pack(">b", n))
|
|
107
|
+
elif n > 0:
|
|
108
|
+
for code, fmt, top in ((0xCC, ">BB", 0xFF), (0xCD, ">BH", 0xFFFF), (0xCE, ">BI", 0xFFFFFFFF),
|
|
109
|
+
(0xCF, ">BQ", 0xFFFFFFFFFFFFFFFF)):
|
|
110
|
+
if n <= top:
|
|
111
|
+
out.append(struct.pack(fmt, code, n))
|
|
112
|
+
return
|
|
113
|
+
raise MsgpackError(f"an int past msgpack's 64 bits: {n}")
|
|
114
|
+
else:
|
|
115
|
+
for code, fmt, low in ((0xD0, ">Bb", -0x80), (0xD1, ">Bh", -0x8000), (0xD2, ">Bi", -0x80000000),
|
|
116
|
+
(0xD3, ">Bq", -0x8000000000000000)):
|
|
117
|
+
if n >= low:
|
|
118
|
+
out.append(struct.pack(fmt, code, n))
|
|
119
|
+
return
|
|
120
|
+
raise MsgpackError(f"an int past msgpack's 64 bits: {n}")
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _pack_head(n: int, out: list, *, fix: tuple | None, sizes: tuple, what: str) -> None:
|
|
124
|
+
if fix is not None and n <= fix[1]:
|
|
125
|
+
out.append(bytes((fix[0] | n,)))
|
|
126
|
+
return
|
|
127
|
+
for code, fmt, top in sizes:
|
|
128
|
+
if n <= top:
|
|
129
|
+
out.append(struct.pack(fmt, code, n))
|
|
130
|
+
return
|
|
131
|
+
raise MsgpackError(f"a {what} of {n}: past msgpack's 32-bit lengths")
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _numpy_map(obj: Any) -> dict | None:
|
|
135
|
+
"""A numpy array or scalar as the map msgpack-numpy writes it, or None for anything else (numpy not imported by
|
|
136
|
+
whoever made `obj` means it holds none)."""
|
|
137
|
+
import sys
|
|
138
|
+
|
|
139
|
+
np = sys.modules.get("numpy")
|
|
140
|
+
if np is None:
|
|
141
|
+
return None
|
|
142
|
+
if isinstance(obj, np.ndarray):
|
|
143
|
+
_readable_dtype(obj.dtype)
|
|
144
|
+
return {b"nd": True, b"type": obj.dtype.str, b"kind": b"", b"shape": list(obj.shape),
|
|
145
|
+
b"data": np.ascontiguousarray(obj).tobytes()}
|
|
146
|
+
if isinstance(obj, (np.bool_, np.number)):
|
|
147
|
+
_readable_dtype(obj.dtype)
|
|
148
|
+
return {b"nd": False, b"type": obj.dtype.str, b"data": np.asarray(obj).tobytes()}
|
|
149
|
+
return None
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _readable_dtype(dtype) -> None:
|
|
153
|
+
if dtype.hasobject or dtype.kind in ("O", "V"):
|
|
154
|
+
raise MsgpackError(f"a numpy array of dtype {dtype} is not carried: only plain numeric, bool and string dtypes "
|
|
155
|
+
"are")
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def unpack(data: bytes) -> Any:
|
|
160
|
+
"""The one msgpack value `data` holds, numpy arrays (msgpack-numpy's or openpi's layout) back as read-only arrays -
|
|
161
|
+
or, where numpy is not installed, as the nested lists of plain values their tolist() gives, what a JSON answer
|
|
162
|
+
holds. MsgpackError: truncated, more than one value, an extension type, nested past MAX_DEPTH, a length past the
|
|
163
|
+
body, or an array whose dtype holds objects."""
|
|
164
|
+
view = memoryview(bytes(data)) if not isinstance(data, bytes) else memoryview(data)
|
|
165
|
+
value, at = _Reader(view).read(0, 0)
|
|
166
|
+
if at != len(view):
|
|
167
|
+
raise MsgpackError(f"{len(view) - at} bytes follow the msgpack value")
|
|
168
|
+
return value
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class _Reader:
|
|
172
|
+
__slots__ = ("buf", "size")
|
|
173
|
+
|
|
174
|
+
def __init__(self, buf: memoryview) -> None:
|
|
175
|
+
self.buf, self.size = buf, len(buf)
|
|
176
|
+
|
|
177
|
+
def _take(self, at: int, n: int) -> int:
|
|
178
|
+
if n < 0 or at + n > self.size:
|
|
179
|
+
raise MsgpackError(f"the body ends inside a value: {n} bytes asked for at {at} of {self.size}")
|
|
180
|
+
return at + n
|
|
181
|
+
|
|
182
|
+
def _unpack(self, fmt: str, at: int) -> tuple:
|
|
183
|
+
end = self._take(at, struct.calcsize(fmt))
|
|
184
|
+
return struct.unpack_from(fmt, self.buf, at), end
|
|
185
|
+
|
|
186
|
+
def read(self, at: int, depth: int) -> tuple[Any, int]:
|
|
187
|
+
if depth > MAX_DEPTH:
|
|
188
|
+
raise MsgpackError(f"nested deeper than {MAX_DEPTH} levels")
|
|
189
|
+
self._take(at, 1)
|
|
190
|
+
code = self.buf[at]
|
|
191
|
+
at += 1
|
|
192
|
+
if code <= 0x7F:
|
|
193
|
+
return code, at
|
|
194
|
+
if code >= 0xE0:
|
|
195
|
+
return code - 0x100, at
|
|
196
|
+
if 0xA0 <= code <= 0xBF:
|
|
197
|
+
return self._str(at, code & 0x1F)
|
|
198
|
+
if 0x90 <= code <= 0x9F:
|
|
199
|
+
return self._list(at, code & 0x0F, depth)
|
|
200
|
+
if 0x80 <= code <= 0x8F:
|
|
201
|
+
return self._map(at, code & 0x0F, depth)
|
|
202
|
+
if code == 0xC0:
|
|
203
|
+
return None, at
|
|
204
|
+
if code == 0xC2:
|
|
205
|
+
return False, at
|
|
206
|
+
if code == 0xC3:
|
|
207
|
+
return True, at
|
|
208
|
+
fixed = _FIXED.get(code)
|
|
209
|
+
if fixed is not None:
|
|
210
|
+
(value,), end = self._unpack(fixed, at)
|
|
211
|
+
return value, end
|
|
212
|
+
sized = _SIZED.get(code)
|
|
213
|
+
if sized is not None:
|
|
214
|
+
kind, fmt = sized
|
|
215
|
+
(n,), at = self._unpack(fmt, at)
|
|
216
|
+
if kind == "str":
|
|
217
|
+
return self._str(at, n)
|
|
218
|
+
if kind == "bin":
|
|
219
|
+
end = self._take(at, n)
|
|
220
|
+
return bytes(self.buf[at:end]), end
|
|
221
|
+
if kind == "list":
|
|
222
|
+
return self._list(at, n, depth)
|
|
223
|
+
return self._map(at, n, depth)
|
|
224
|
+
if code in _EXT or code == 0xC1:
|
|
225
|
+
raise MsgpackError(f"msgpack type 0x{code:02x} is not read here (extension types and the unused code)")
|
|
226
|
+
raise MsgpackError(f"not a msgpack type: 0x{code:02x}")
|
|
227
|
+
|
|
228
|
+
def _str(self, at: int, n: int) -> tuple[str, int]:
|
|
229
|
+
end = self._take(at, n)
|
|
230
|
+
try:
|
|
231
|
+
return str(self.buf[at:end], "utf-8"), end
|
|
232
|
+
except UnicodeDecodeError:
|
|
233
|
+
raise MsgpackError("a string that is not UTF-8") from None
|
|
234
|
+
|
|
235
|
+
def _list(self, at: int, n: int, depth: int) -> tuple[list, int]:
|
|
236
|
+
if n > self.size - at:
|
|
237
|
+
raise MsgpackError(f"a list of {n} items where {self.size - at} bytes are left")
|
|
238
|
+
out = []
|
|
239
|
+
for _ in range(n):
|
|
240
|
+
value, at = self.read(at, depth + 1)
|
|
241
|
+
out.append(value)
|
|
242
|
+
return out, at
|
|
243
|
+
|
|
244
|
+
def _map(self, at: int, n: int, depth: int) -> tuple[Any, int]:
|
|
245
|
+
if 2 * n > self.size - at:
|
|
246
|
+
raise MsgpackError(f"a map of {n} entries where {self.size - at} bytes are left")
|
|
247
|
+
out: dict = {}
|
|
248
|
+
for _ in range(n):
|
|
249
|
+
key, at = self.read(at, depth + 1)
|
|
250
|
+
value, at = self.read(at, depth + 1)
|
|
251
|
+
try:
|
|
252
|
+
out[key] = value
|
|
253
|
+
except TypeError:
|
|
254
|
+
raise MsgpackError(f"a map key that cannot be a dict key: {type(key).__name__}") from None
|
|
255
|
+
return _array_of(out), at
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
_FIXED = {0xCA: ">f", 0xCB: ">d", 0xCC: ">B", 0xCD: ">H", 0xCE: ">I", 0xCF: ">Q", 0xD0: ">b", 0xD1: ">h", 0xD2: ">i",
|
|
259
|
+
0xD3: ">q"}
|
|
260
|
+
_SIZED = {0xD9: ("str", ">B"), 0xDA: ("str", ">H"), 0xDB: ("str", ">I"), 0xC4: ("bin", ">B"), 0xC5: ("bin", ">H"),
|
|
261
|
+
0xC6: ("bin", ">I"), 0xDC: ("list", ">H"), 0xDD: ("list", ">I"), 0xDE: ("map", ">H"), 0xDF: ("map", ">I")}
|
|
262
|
+
_EXT = {0xC7, 0xC8, 0xC9, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8}
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _field(m: dict, name: str):
|
|
266
|
+
"""A field of an array's map, its key as msgpack-numpy writes it (bytes) or as a packer without bin types does."""
|
|
267
|
+
return m[name.encode()] if name.encode() in m else m.get(name)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _text(value) -> str:
|
|
271
|
+
return value.decode("ascii", "replace") if isinstance(value, bytes) else str(value)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _array_of(m: dict) -> Any:
|
|
275
|
+
"""`m` as the numpy array or scalar it stands for when it has every field of one in msgpack-numpy's or openpi's
|
|
276
|
+
layout, else `m` as it is - a dict of the user's that happens to hold a key "nd" stays a dict."""
|
|
277
|
+
nd, openpi, generic = _field(m, "nd"), _field(m, "__ndarray__"), _field(m, "__npgeneric__")
|
|
278
|
+
has = lambda *names: all(n.encode() in m or n in m for n in names)
|
|
279
|
+
if nd is True and has("type", "shape", "data"):
|
|
280
|
+
_dtype(_field(m, "type"))
|
|
281
|
+
if _field(m, "kind") not in (None, b"", ""):
|
|
282
|
+
raise MsgpackError("a structured numpy array is not carried: only plain numeric, bool and string dtypes "
|
|
283
|
+
"are")
|
|
284
|
+
return _array(_field(m, "type"), _field(m, "shape"), _field(m, "data"))
|
|
285
|
+
if openpi is True and has("dtype", "shape", "data"):
|
|
286
|
+
return _array(_field(m, "dtype"), _field(m, "shape"), _field(m, "data"))
|
|
287
|
+
if nd is False and has("type", "data"):
|
|
288
|
+
dtype, data = _dtype(_field(m, "type")), _field(m, "data")
|
|
289
|
+
if not isinstance(data, bytes) or len(data) != dtype.itemsize:
|
|
290
|
+
raise MsgpackError(f"a numpy {dtype} scalar whose data is not {dtype.itemsize} bytes")
|
|
291
|
+
np = _numpy()
|
|
292
|
+
return _plain_items(dtype, data, 1)[0] if np is None else np.frombuffer(data, dtype=dtype, count=1)[0]
|
|
293
|
+
if generic is True and has("dtype", "data"):
|
|
294
|
+
dtype, value = _dtype(_field(m, "dtype")), _field(m, "data")
|
|
295
|
+
if not isinstance(value, (bool, int, float, str, bytes)):
|
|
296
|
+
raise MsgpackError("a numpy scalar's map whose data is not a plain value")
|
|
297
|
+
return value if _numpy() is None else dtype.type(value)
|
|
298
|
+
return m
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _array(descr, shape, data):
|
|
302
|
+
dtype = _dtype(descr)
|
|
303
|
+
if not isinstance(shape, list) or not all(isinstance(d, int) and not isinstance(d, bool) and d >= 0
|
|
304
|
+
for d in shape) or not isinstance(data, bytes):
|
|
305
|
+
raise MsgpackError("a numpy array's map without a shape of non-negative ints or its data as bytes")
|
|
306
|
+
if len(shape) > _MAX_DIMS:
|
|
307
|
+
raise MsgpackError(f"a numpy array of {len(shape)} dimensions: numpy's own are {_MAX_DIMS} at most")
|
|
308
|
+
count = 1
|
|
309
|
+
for d in shape:
|
|
310
|
+
count *= d
|
|
311
|
+
if len(data) != count * dtype.itemsize:
|
|
312
|
+
raise MsgpackError(f"a numpy array of shape {shape} and dtype {dtype} is {count * dtype.itemsize} bytes; its "
|
|
313
|
+
f"data is {len(data)}")
|
|
314
|
+
np = _numpy()
|
|
315
|
+
if np is None:
|
|
316
|
+
return _nested(_plain_items(dtype, data, count), shape)
|
|
317
|
+
return np.frombuffer(data, dtype=dtype, count=count).reshape(shape)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
_MAX_DIMS = 64
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _numpy():
|
|
325
|
+
"""numpy, or None where it is not installed: its arrays are then read as plain values (_Plain)."""
|
|
326
|
+
try:
|
|
327
|
+
import numpy
|
|
328
|
+
except ImportError:
|
|
329
|
+
return None
|
|
330
|
+
return numpy
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _dtype(descr):
|
|
334
|
+
"""`descr`, an array's type string, as numpy's dtype - or, where numpy is not installed, as a _Plain read here.
|
|
335
|
+
MsgpackError for one that holds objects, a structured one, and one that is not a type string."""
|
|
336
|
+
if not isinstance(descr, (str, bytes)):
|
|
337
|
+
raise MsgpackError("a numpy array's dtype is not a type string: a structured array is not carried, only "
|
|
338
|
+
"plain numeric, bool and string dtypes are")
|
|
339
|
+
np = _numpy()
|
|
340
|
+
if np is None:
|
|
341
|
+
return _plain(_text(descr))
|
|
342
|
+
try:
|
|
343
|
+
dtype = np.dtype(_text(descr))
|
|
344
|
+
except TypeError:
|
|
345
|
+
raise MsgpackError(f"not a numpy dtype: {_text(descr)!r}") from None
|
|
346
|
+
_readable_dtype(dtype)
|
|
347
|
+
return dtype
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
class _Plain(NamedTuple):
|
|
357
|
+
"""An array's type string ('<f4', '|u1', '<U5' - numpy's dtype.str) read without numpy: struct's byte order, the
|
|
358
|
+
kind (numpy's letter) and one item's size in bytes."""
|
|
359
|
+
text: str
|
|
360
|
+
order: str
|
|
361
|
+
kind: str
|
|
362
|
+
itemsize: int
|
|
363
|
+
|
|
364
|
+
def __str__(self) -> str:
|
|
365
|
+
return self.text
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
_STRUCT = {("b", 1): "?", ("i", 1): "b", ("i", 2): "h", ("i", 4): "i", ("i", 8): "q", ("u", 1): "B", ("u", 2): "H",
|
|
371
|
+
("u", 4): "I", ("u", 8): "Q", ("f", 2): "e", ("f", 4): "f", ("f", 8): "d", ("c", 8): "f", ("c", 16): "d"}
|
|
372
|
+
_TYPE = re.compile(r"([<>|=]?)([A-Za-z])(.*)", re.DOTALL)
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _plain(text: str) -> _Plain:
|
|
376
|
+
"""MsgpackError for a type string that holds objects or is structured - refused as with numpy - one whose items
|
|
377
|
+
are no bytes at all, and one only numpy reads (a long double, a date, a time span)."""
|
|
378
|
+
match = _TYPE.fullmatch(text)
|
|
379
|
+
order, kind, size = match.groups() if match else ("", "", "")
|
|
380
|
+
if kind in ("O", "V"):
|
|
381
|
+
raise MsgpackError(f"a numpy array of dtype {text} is not carried: only plain numeric, bool and string dtypes "
|
|
382
|
+
"are")
|
|
383
|
+
n = int(size) if size.isascii() and size.isdigit() else -1
|
|
384
|
+
if kind in ("S", "U") and n == 0:
|
|
385
|
+
raise MsgpackError(f"a numpy array of dtype {text}, whose items are 0 bytes")
|
|
386
|
+
itemsize = 4 * n if kind == "U" else n
|
|
387
|
+
if not ((kind in ("S", "U") and n > 0) or (kind, itemsize) in _STRUCT):
|
|
388
|
+
raise MsgpackError(f"this body holds a numpy array of dtype {text}, which only numpy reads, and numpy is not "
|
|
389
|
+
"installed here")
|
|
390
|
+
return _Plain(text, "<" if order in ("", "|") else order, kind, itemsize)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _plain_items(t: _Plain, data: bytes, count: int) -> list:
|
|
394
|
+
"""`count` items of type `t` from `data`, as tolist() gives them: ints, floats, complexes and bools; bytes and
|
|
395
|
+
strs with their trailing NULs cut, as numpy cuts them."""
|
|
396
|
+
size = t.itemsize
|
|
397
|
+
if t.kind == "S":
|
|
398
|
+
return [data[i:i + size].rstrip(b"\x00") for i in range(0, count * size, size)]
|
|
399
|
+
if t.kind == "U":
|
|
400
|
+
big = t.order == ">" or (t.order == "=" and sys.byteorder == "big")
|
|
401
|
+
try:
|
|
402
|
+
return [str(data[i:i + size], "utf-32-be" if big else "utf-32-le").rstrip("\x00")
|
|
403
|
+
for i in range(0, count * size, size)]
|
|
404
|
+
except UnicodeDecodeError:
|
|
405
|
+
raise MsgpackError(f"a numpy array of dtype {t} whose data is not UTF-32") from None
|
|
406
|
+
code = _STRUCT[(t.kind, size)]
|
|
407
|
+
if t.kind == "c":
|
|
408
|
+
parts = struct.unpack_from(f"{t.order}{2 * count}{code}", data)
|
|
409
|
+
return [complex(parts[i], parts[i + 1]) for i in range(0, 2 * count, 2)]
|
|
410
|
+
return list(struct.unpack_from(f"{t.order}{count}{code}", data))
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def _nested(flat: list, shape: list) -> Any:
|
|
414
|
+
"""`flat`, in C order, as the nested lists of `shape`, as tolist() nests them - a 0-d array's one value as itself."""
|
|
415
|
+
if not shape:
|
|
416
|
+
return flat[0]
|
|
417
|
+
if len(shape) == 1:
|
|
418
|
+
return flat
|
|
419
|
+
step = len(flat) // shape[0] if shape[0] else 0
|
|
420
|
+
return [_nested(flat[i * step:(i + 1) * step], shape[1:]) for i in range(shape[0])]
|
|
@@ -379,7 +379,7 @@
|
|
|
379
379
|
],
|
|
380
380
|
"title": "SeqFlags",
|
|
381
381
|
"type": "integer",
|
|
382
|
-
"x-contract-version": "0.5.
|
|
382
|
+
"x-contract-version": "0.5.21",
|
|
383
383
|
"x-flag-names": {
|
|
384
384
|
"CHECK": 65536,
|
|
385
385
|
"INFER": 16,
|
|
@@ -628,5 +628,5 @@
|
|
|
628
628
|
],
|
|
629
629
|
"title": "BenchmarkSpec",
|
|
630
630
|
"type": "object",
|
|
631
|
-
"x-contract-version": "0.5.
|
|
631
|
+
"x-contract-version": "0.5.21"
|
|
632
632
|
}
|