sequence-base 0.7.0__tar.gz → 0.7.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. {sequence_base-0.7.0 → sequence_base-0.7.2}/PKG-INFO +1 -1
  2. {sequence_base-0.7.0 → sequence_base-0.7.2}/pyproject.toml +1 -1
  3. sequence_base-0.7.2/sequence_base/codec/blocks.py +248 -0
  4. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/constants/limits.py +4 -0
  5. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/constants/version.py +1 -1
  6. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/deployments.py +6 -3
  7. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/observe.py +13 -2
  8. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/obs/log.py +14 -5
  9. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ActRequest.json +1 -1
  10. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ActResponse.json +1 -1
  11. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ActionChunk.json +1 -1
  12. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ActionContract.json +1 -1
  13. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ActionStep.json +1 -1
  14. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/AdapterPair.json +1 -1
  15. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/AddLocal.json +1 -1
  16. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/Autoscaler.json +1 -1
  17. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/BenchmarkRegisterRequest.json +1 -1
  18. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/BenchmarkSpec.json +2 -2
  19. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/BenchmarkStatusResponse.json +1 -1
  20. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/BootstrapResponse.json +1 -1
  21. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/Camera.json +1 -1
  22. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/CodeBundle.json +1 -1
  23. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/Concurrency.json +1 -1
  24. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ConnectRequest.json +1 -1
  25. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ConnectResponse.json +1 -1
  26. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/DeploymentAccepted.json +1 -1
  27. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/DeploymentPolicy.json +1 -1
  28. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/DeploymentRequest.json +2 -3
  29. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/DeploymentStatusResponse.json +1 -1
  30. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ErrorBody.json +1 -1
  31. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ErrorCodes.json +1 -1
  32. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/Estimate.json +1 -1
  33. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/EvalDryRunResponse.json +1 -1
  34. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/EvalReport.json +1 -1
  35. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/EvalRequest.json +1 -1
  36. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/EvalResponse.json +1 -1
  37. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ImageFrame.json +1 -1
  38. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ImageSpec.json +1 -1
  39. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ImageStepAddLocal.json +1 -1
  40. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ImageStepApt.json +1 -1
  41. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ImageStepEnv.json +1 -1
  42. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ImageStepProject.json +1 -1
  43. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ImageStepRun.json +1 -1
  44. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ImageStepTemplateFiles.json +1 -1
  45. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ImageStepUvPip.json +1 -1
  46. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/InferParam.json +1 -1
  47. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/Lease.json +1 -1
  48. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/MetricAgg.json +1 -1
  49. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/Observation.json +1 -1
  50. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ObservationContract.json +1 -1
  51. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ObserveRequest.json +10 -1
  52. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ObserveResponse.json +1 -1
  53. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/PlanAppend.json +1 -1
  54. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/PlanAppended.json +1 -1
  55. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/PlannerMember.json +1 -1
  56. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/PlannerObservation.json +1 -1
  57. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/PlannerObserve.json +1 -1
  58. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/PlannerObserved.json +1 -1
  59. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/PlannerPlanIn.json +1 -1
  60. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/PlannerPlans.json +1 -1
  61. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/PlannerPlansWritten.json +1 -1
  62. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/PlannerRobot.json +1 -1
  63. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/PlannerRound.json +1 -1
  64. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/PlannerRoundRequest.json +1 -1
  65. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/PlannerSpec.json +1 -1
  66. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/PolicyRegistration.json +2 -2
  67. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/PolicySpec.json +1 -1
  68. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/Project.json +1 -1
  69. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/Proprioception.json +1 -1
  70. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/ReplaySpec.json +1 -1
  71. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/Resources.json +2 -3
  72. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/RolloutResult.json +1 -1
  73. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/SecretCreateRequest.json +1 -1
  74. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/SecretInfo.json +1 -1
  75. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/SecretRef.json +1 -1
  76. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/SeqFlags.json +1 -1
  77. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/SettingsChange.json +1 -1
  78. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/SettingsChanged.json +1 -1
  79. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/SettingsView.json +1 -1
  80. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/Standby.json +1 -1
  81. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/TaskAgg.json +1 -1
  82. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/TemplateFiles.json +1 -1
  83. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/Tensors.json +1 -1
  84. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/Timeouts.json +1 -1
  85. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/UploadTarget.json +1 -1
  86. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/VolumeFile.json +1 -1
  87. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/VolumeInfo.json +1 -1
  88. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/VolumeMount.json +1 -1
  89. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/VolumeRef.json +1 -1
  90. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/VolumeUploadRequest.json +1 -1
  91. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/VolumeUploadResponse.json +1 -1
  92. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schemas/WarmingError.json +1 -1
  93. {sequence_base-0.7.0 → sequence_base-0.7.2}/.gitignore +0 -0
  94. {sequence_base-0.7.0 → sequence_base-0.7.2}/README.md +0 -0
  95. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/__init__.py +0 -0
  96. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/auth.py +0 -0
  97. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/by_name.py +0 -0
  98. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/codec/__init__.py +0 -0
  99. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/codec/compressed.py +0 -0
  100. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/codec/image.py +0 -0
  101. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/codec/marked.py +0 -0
  102. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/constants/__init__.py +0 -0
  103. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/constants/cards.py +0 -0
  104. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/constants/headers.py +0 -0
  105. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/construct.py +0 -0
  106. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/__init__.py +0 -0
  107. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/accounts.py +0 -0
  108. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/act.py +0 -0
  109. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/adapters.py +0 -0
  110. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/benchmarks.py +0 -0
  111. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/connect.py +0 -0
  112. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/errors.py +0 -0
  113. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/eval.py +0 -0
  114. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/flags.py +0 -0
  115. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/io.py +0 -0
  116. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/own_io.py +0 -0
  117. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/planner.py +0 -0
  118. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/policy.py +0 -0
  119. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/retired.py +0 -0
  120. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/settings.py +0 -0
  121. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/templates.py +0 -0
  122. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/volumes.py +0 -0
  123. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/contract/wire.py +0 -0
  124. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/episode.py +0 -0
  125. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/errors/__init__.py +0 -0
  126. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/errors/transport.py +0 -0
  127. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/msgpack_io.py +0 -0
  128. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/obs/__init__.py +0 -0
  129. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/obs/rid.py +0 -0
  130. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/presets/__init__.py +0 -0
  131. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/presets/frames.py +0 -0
  132. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/presets/robot_arm.py +0 -0
  133. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/schema_export.py +0 -0
  134. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/telemetry/__init__.py +0 -0
  135. {sequence_base-0.7.0 → sequence_base-0.7.2}/sequence_base/telemetry/events.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: sequence-base
3
- Version: 0.7.0
3
+ Version: 0.7.2
4
4
  Summary: Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness.
5
5
  Project-URL: Changelog, https://generalsequences.com/docs/changelog/
6
6
  Requires-Python: >=3.10
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "sequence-base"
3
- version = "0.7.0"
3
+ version = "0.7.2"
4
4
  description = "Shared foundation for the Sequences platform — wire contract, correlation, error taxonomy, constants. Used by the robot SDK, gateway, and container harness."
5
5
  requires-python = ">=3.10"
6
6
  dependencies = ["pydantic>=2.0"]
@@ -0,0 +1,248 @@
1
+ """
2
+ Observe blocks: an episode's large arrays, bytes and text crossing to its worker once - what a robot's SDK puts in place
3
+ of one whose content its worker already holds, and what the worker turns back - so a sliding window of frames, or a
4
+ large input sent again with every observation, goes up once.
5
+
6
+
7
+ In one sentence: the SDK, packing an observation for a worker that names FORMAT, cuts each unmarked numpy array, bytes
8
+ or str of MIN_BLOCK_BYTES or more into blocks - an array stacked along its first axis, 2 to SPLIT_MAX slices each of
9
+ MIN_BLOCK_BYTES or more, a block a slice - each named by the first ID_BYTES of its sha256 (block_id) → puts a reference
10
+ in its place (encode) → its post carries the content of each block its worker does not hold (ObserveRequest.blocks), as
11
+ both sides count what the worker holds (Retention) → the worker keeps each episode's blocks and, as each observation's
12
+ turn comes, in order, turns its references back into what the robot built (decode): arrays read-only, as msgpack_io
13
+ reads them, bytes, str - so its policy is handed the bytes the robot sent.
14
+
15
+ A reference, inside the observation's msgpack (`blocks`: the ids, each ID_BYTES bin, in order):
16
+ {"__seq__": "array", "v": 1, "type": numpy's string for its dtype, "shape": [...], "blocks": [id, ...]}
17
+ {"__seq__": "bytes", "v": 1, "blocks": [id]}
18
+ {"__seq__": "str", "v": 1, "blocks": [id]} the text's UTF-8
19
+ WHY by content, not by key or place: a window's frames change places as it slides, and the user names its own keys;
20
+ only the bytes say a frame was sent before. WHY cut a stacked window by its first axis: as it slides, the stacked array
21
+ is new every time - a slice of it is not. WHY the robot decides what is held, by a rule both count alike: asking the
22
+ worker would add a round trip to every observation, which the user ruled out ("Need to make sure this is parallel.").
23
+ A worker that lacks a block after all - restarted, another one, past what it keeps - answers frames_missing, and the
24
+ robot sends again with every block's content: never a wrong observation, only that once more bytes.
25
+ WHY the observation format's reserved key: marked (codec/marked.py) and compressed (codec/compressed.py) items already
26
+ take a map with "__seq__"; a reference is never inside either, and neither is cut into blocks.
27
+
28
+ encoded, contents, used = encode(observation) # the SDK, packing it
29
+ observation = decode(encoded, table) # the worker, at its turn: table maps id -> content
30
+ """
31
+ from __future__ import annotations
32
+
33
+ import hashlib
34
+ from collections import deque
35
+ from typing import Any, Iterable, Mapping
36
+
37
+ from sequence_base.codec.compressed import is_compressed
38
+ from sequence_base.codec.marked import MARK, is_marked
39
+ from sequence_base.contract.observe import BLOCKS_FORMAT
40
+
41
+ __all__ = ["FORMAT", "ID_BYTES", "KEEP_OBSERVATIONS", "KINDS", "MIN_BLOCK_BYTES", "SPLIT_MAX", "VERSION", "BlocksError",
42
+ "MissingBlock", "Retention", "block_id", "cut", "decode", "encode", "ids_of", "is_reference"]
43
+
44
+
45
+
46
+ FORMAT = BLOCKS_FORMAT
47
+ KINDS = ("array", "bytes", "str")
48
+ VERSION = 1
49
+ ID_BYTES = 16
50
+ MIN_BLOCK_BYTES = 16 * 1024
51
+ SPLIT_MAX = 64
52
+ KEEP_OBSERVATIONS = 8
53
+
54
+
55
+ class BlocksError(ValueError):
56
+ """A reference that does not read: a kind or version not taken, ids that are not ID_BYTES bin, a shape and dtype
57
+ that do not match the blocks' bytes."""
58
+
59
+
60
+ class MissingBlock(BlocksError):
61
+ """A reference naming a block its worker does not hold (`block`): its robot sends it again, with the content."""
62
+
63
+ def __init__(self, block: bytes) -> None:
64
+ super().__init__(f"a block this worker does not hold: {bytes(block).hex()}")
65
+ self.block = bytes(block)
66
+
67
+
68
+ def block_id(data: bytes | bytearray | memoryview) -> bytes:
69
+ """A block's name: the first ID_BYTES of its content's sha256 - whose OpenSSL implementation runs on a CPU's SHA
70
+ instructions (Apple M, ARMv8.2 with crypto - Jetson Orin's generation -, x86 with SHA-NI) about three times as
71
+ fast as blake2b.
72
+ """
73
+ return hashlib.sha256(data).digest()[:ID_BYTES]
74
+
75
+
76
+ def is_reference(value: Any) -> bool:
77
+ return isinstance(value, dict) and value.get(MARK) in KINDS
78
+
79
+
80
+ def cut(value: Any) -> "list[bytes] | None":
81
+ """The blocks a value is sent as - its content, or an array stacked along its first axis a slice each - or None for
82
+ one that is no block: smaller than MIN_BLOCK_BYTES, of another type, or an array whose bytes do not stand for its
83
+ values (numpy's object dtype)."""
84
+ if isinstance(value, (bytes, bytearray, memoryview)):
85
+ data = bytes(value)
86
+ return [data] if len(data) >= MIN_BLOCK_BYTES else None
87
+ if isinstance(value, str):
88
+ data = value.encode("utf-8")
89
+ return [data] if len(data) >= MIN_BLOCK_BYTES else None
90
+ np = _numpy()
91
+ if np is None or not isinstance(value, np.ndarray) or value.dtype.hasobject or value.nbytes < MIN_BLOCK_BYTES:
92
+ return None
93
+ data = np.ascontiguousarray(value)
94
+ if data.ndim >= 2 and 2 <= data.shape[0] <= SPLIT_MAX and data.nbytes // data.shape[0] >= MIN_BLOCK_BYTES:
95
+ return [data[i].tobytes() for i in range(data.shape[0])]
96
+ return [data.tobytes()]
97
+
98
+
99
+ def encode(value: Any) -> "tuple[Any, dict[bytes, bytes], list[bytes]]":
100
+ """`value` with every block it holds - in dicts, lists and tuples at any depth, never inside a marked or compressed
101
+ item - put as its reference (tuples become lists, as msgpack writes them); the blocks' contents by id; and every id
102
+ it used, in order, once each. A value that holds none is handed back as it is, the same object."""
103
+ contents: dict[bytes, bytes] = {}
104
+ used: list[bytes] = []
105
+
106
+ def walk(v: Any) -> Any:
107
+ if isinstance(v, dict):
108
+ if is_marked(v) or is_compressed(v) or is_reference(v):
109
+ return v
110
+ out = {k: walk(x) for k, x in v.items()}
111
+ return out if any(out[k] is not v[k] for k in v) else v
112
+ if isinstance(v, (list, tuple)):
113
+ out = [walk(x) for x in v]
114
+ return out if isinstance(v, tuple) or any(a is not b for a, b in zip(out, v)) else v
115
+ parts = cut(v)
116
+ if parts is None:
117
+ return v
118
+ ids = []
119
+ for part in parts:
120
+ bid = block_id(part)
121
+ if bid not in contents:
122
+ contents[bid] = part
123
+ used.append(bid)
124
+ ids.append(bid)
125
+ if isinstance(v, str):
126
+ return dict([(MARK, "str"), ("v", VERSION), ("blocks", ids)])
127
+ if isinstance(v, (bytes, bytearray, memoryview)):
128
+ return dict([(MARK, "bytes"), ("v", VERSION), ("blocks", ids)])
129
+ return dict([(MARK, "array"), ("v", VERSION), ("type", v.dtype.str), ("shape", list(v.shape)),
130
+ ("blocks", ids)])
131
+
132
+ return walk(value), contents, used
133
+
134
+
135
+ def ids_of(value: Any) -> list[bytes]:
136
+ """Every block id `value`'s references name, in order, once each - what turning it back needs."""
137
+ seen: dict[bytes, None] = {}
138
+
139
+ def walk(v: Any) -> None:
140
+ if is_reference(v):
141
+ for bid in _ids(v):
142
+ seen.setdefault(bid, None)
143
+ elif isinstance(v, dict) and not (is_marked(v) or is_compressed(v)):
144
+ for x in v.values():
145
+ walk(x)
146
+ elif isinstance(v, (list, tuple)):
147
+ for x in v:
148
+ walk(x)
149
+
150
+ walk(value)
151
+ return list(seen)
152
+
153
+
154
+ def decode(value: Any, table: Mapping[bytes, Any]) -> Any:
155
+ """`value` with every reference turned back from `table` (id → the block's content) into what the robot built: an
156
+ array read-only, its dtype and shape the reference's - a stacked one put together once, the only copy - bytes, or
157
+ str. The same objects where it holds none. MissingBlock for a block `table` lacks; BlocksError for a reference that
158
+ does not read."""
159
+ def walk(v: Any) -> Any:
160
+ if isinstance(v, dict):
161
+ if is_reference(v):
162
+ return _value(v, table)
163
+ if is_marked(v) or is_compressed(v):
164
+ return v
165
+ out = {k: walk(x) for k, x in v.items()}
166
+ return out if any(out[k] is not v[k] for k in v) else v
167
+ if isinstance(v, list):
168
+ out = [walk(x) for x in v]
169
+ return out if any(a is not b for a, b in zip(out, v)) else v
170
+ return v
171
+
172
+ return walk(value)
173
+
174
+
175
+ def _ids(ref: dict) -> list[bytes]:
176
+ if ref.get("v") != VERSION:
177
+ raise BlocksError(f"a {ref.get(MARK)} reference of version {ref.get('v')!r}: {VERSION} is read")
178
+ ids = ref.get("blocks")
179
+ if not isinstance(ids, list) or not ids or not all(isinstance(b, (bytes, bytearray)) and len(b) == ID_BYTES
180
+ for b in ids):
181
+ raise BlocksError(f"a {ref.get(MARK)} reference whose blocks are not a list of {ID_BYTES}-byte ids")
182
+ return [bytes(b) for b in ids]
183
+
184
+
185
+ def _value(ref: dict, table: Mapping[bytes, Any]) -> Any:
186
+ kind, ids = ref[MARK], _ids(ref)
187
+ parts = []
188
+ for bid in ids:
189
+ part = table.get(bid)
190
+ if part is None:
191
+ raise MissingBlock(bid)
192
+ parts.append(part)
193
+ if kind in ("bytes", "str"):
194
+ if len(parts) != 1:
195
+ raise BlocksError(f"a {kind} reference of {len(parts)} blocks: one is read")
196
+ return bytes(parts[0]) if kind == "bytes" else bytes(parts[0]).decode("utf-8")
197
+ np = _numpy()
198
+ if np is None:
199
+ raise BlocksError("an array reference, and no numpy to make it with")
200
+ from sequence_base.msgpack_io import MsgpackError, _dtype
201
+
202
+ shape = ref.get("shape")
203
+ if not isinstance(shape, list) or not all(isinstance(d, int) and not isinstance(d, bool) and d >= 0 for d in shape):
204
+ raise BlocksError("an array reference whose shape is not a list of non-negative ints")
205
+ try:
206
+ dtype = _dtype(ref.get("type"))
207
+ except MsgpackError as exc:
208
+ raise BlocksError(f"an array reference: {exc}") from None
209
+ data = parts[0] if len(parts) == 1 else b"".join(bytes(p) for p in parts)
210
+ if dtype.itemsize * int(np.prod(shape, dtype=np.int64)) != len(data):
211
+ raise BlocksError(f"an array reference of {dtype} and shape {shape} whose blocks hold {len(data)} bytes")
212
+ out = np.frombuffer(data, dtype=dtype).reshape(shape)
213
+ out.flags.writeable = False
214
+ return out
215
+
216
+
217
+ class Retention:
218
+ """The blocks an episode's worker holds, as the robot and the worker both count them: those the last `keep`
219
+ observations that used any block used, in the observations' order - an observation of no block (its joints alone,
220
+ an instruction) moves nothing. The robot sends the content of each block an observation uses that is not held
221
+ (holds) before it; the worker, turning each observation back in order, keeps what is held after it (use)."""
222
+
223
+ def __init__(self, keep: int = KEEP_OBSERVATIONS) -> None:
224
+ self._recent: deque = deque(maxlen=keep)
225
+
226
+ def holds(self, bid: bytes) -> bool:
227
+ return any(bid in used for used in self._recent)
228
+
229
+ def held(self) -> set:
230
+ return set().union(*self._recent) if self._recent else set()
231
+
232
+ def use(self, ids: Iterable[bytes]) -> set:
233
+ """An observation's blocks, in its turn: the ids held no more after it (its worker drops their contents)."""
234
+ ids = frozenset(ids)
235
+ if not ids:
236
+ return set()
237
+ before = self.held()
238
+ self._recent.append(ids)
239
+ return before - self.held()
240
+
241
+
242
+ def _numpy():
243
+ """numpy where it is installed - a machine without it holds no array to cut."""
244
+ try:
245
+ import numpy as np
246
+ except ImportError:
247
+ return None
248
+ return np
@@ -37,3 +37,7 @@ MAX_STREAM_FRAMES = 1024
37
37
  MAX_STREAM_BYTES = 64 * 1024 * 1024
38
38
  MAX_WORKER_STREAM_BYTES = 1024 * 1024 * 1024
39
39
  STREAM_WAIT_S = 3.0
40
+
41
+
42
+
43
+ MAX_WORKER_BLOCK_BYTES = 512 * 1024 * 1024
@@ -4,7 +4,7 @@
4
4
 
5
5
 
6
6
 
7
- CONTRACT_VERSION = "0.5.35"
7
+ CONTRACT_VERSION = "0.5.36"
8
8
 
9
9
 
10
10
 
@@ -68,9 +68,12 @@ class Resources(BaseModel):
68
68
 
69
69
 
70
70
 
71
- region: str | None = Field(default=None, max_length=32, json_schema_extra={"deprecated": True},
72
- description="Not accepted: a deployment's workers run wherever there are cards, so a "
73
- "region is refused. Omit it (an older SDK still sends it).")
71
+
72
+
73
+ region: str | None = Field(default=None, max_length=32,
74
+ description='Where its workers run: "auto" (the default), each robot\'s in the region '
75
+ 'nearest it, or "us-west" / "us-east", every worker there alone. Any other '
76
+ 'value is refused.')
74
77
 
75
78
  @model_validator(mode="after")
76
79
  def _limits_cover_requests(self) -> "Resources":
@@ -23,8 +23,8 @@ from typing import Any
23
23
 
24
24
  from pydantic import BaseModel, ConfigDict, Field
25
25
 
26
- __all__ = ["FORMAT", "FRAMES_MISSING", "FRAMES_OVER_LIMIT", "ObserveFrame", "ObserveRequest", "ObserveResponse",
27
- "ZSTD_FORMAT"]
26
+ __all__ = ["BLOCKS_FORMAT", "FORMAT", "FRAMES_MISSING", "FRAMES_OVER_LIMIT", "ObserveFrame", "ObserveRequest",
27
+ "ObserveResponse", "ZSTD_FORMAT"]
28
28
 
29
29
 
30
30
 
@@ -38,6 +38,12 @@ ZSTD_FORMAT = "x-seq-zstd/1"
38
38
 
39
39
 
40
40
 
41
+
42
+ BLOCKS_FORMAT = "x-seq-blocks/1"
43
+
44
+
45
+
46
+
41
47
  FRAMES_MISSING = "frames_missing"
42
48
  FRAMES_OVER_LIMIT = "frames_over_limit"
43
49
 
@@ -62,6 +68,11 @@ class ObserveRequest(BaseModel):
62
68
  skipped: list[int] = Field(default_factory=list, description="Numbers left out on purpose, never to come")
63
69
 
64
70
 
71
+
72
+ blocks: dict[bytes, Any] = Field(default_factory=dict,
73
+ description="Block contents by id, for the references in the frames")
74
+
75
+
65
76
  class ObserveResponse(BaseModel):
66
77
  """200 from POST /observe: the highest number up to which the worker holds every observation of the episode, or has
67
78
  been told one is skipped (-1: none yet). A number sent again is answered the same, never kept twice."""
@@ -1,12 +1,13 @@
1
1
  """
2
2
  Structured logging — one line per event: `seq ev=<what> rid=<which> k=v ...`, greppable, flushed.
3
- Deliberately `print` (not `logging`): in a container and in a bare worker, stdout IS the log.
3
+ Deliberately stdout (not `logging`): in a container and in a bare worker, stdout IS the log.
4
4
  {from general-sequences obs.py}
5
5
  """
6
6
  from __future__ import annotations
7
7
 
8
8
  import os
9
9
  import sys
10
+ import threading
10
11
  import time
11
12
 
12
13
  from sequence_base.obs.rid import clean, clean_text
@@ -20,20 +21,28 @@ from sequence_base.obs.rid import clean, clean_text
20
21
  TEXT_FIELDS = frozenset({"error", "warning"})
21
22
 
22
23
 
24
+
25
+
26
+ _WRITING = threading.Lock()
27
+
28
+
23
29
  def enabled() -> bool:
24
30
  """On by default — a fleet that can't be debugged is worse than a noisy one. SEQ_LOG=0 quiets it."""
25
31
  return os.environ.get("SEQ_LOG", "1") != "0"
26
32
 
27
33
 
28
34
  def log(ev: str, rid: str | None = None, **fields: object) -> None:
29
- """Print one event. `seq` first so a grep finds our lines among a vendor's; `ev` a dotted
30
- noun.verb that narrows by prefix; `rid` threads the call. flush=True so a line isn't lost when a
31
- worker is killed before its 30 s log-tail ships."""
35
+ """Write one event as one line. `seq` first so a grep finds our lines among a vendor's; `ev` a
36
+ dotted noun.verb that narrows by prefix; `rid` threads the call. Flushed so a line isn't lost when a
37
+ worker is killed before its 30 s log-tail ships; whole, so two threads' lines never share one."""
32
38
  parts = [f"seq ev={clean(ev)}"]
33
39
  if rid:
34
40
  parts.append(f"rid={clean(rid)}")
35
41
  parts.extend(f"{k}={clean_text(v) if k in TEXT_FIELDS else clean(v)}" for k, v in fields.items() if v is not None)
36
- print(" ".join(parts), file=sys.stdout, flush=True)
42
+ line = " ".join(parts) + "\n"
43
+ with _WRITING:
44
+ sys.stdout.write(line)
45
+ sys.stdout.flush()
37
46
 
38
47
 
39
48
  class Timer:
@@ -66,5 +66,5 @@
66
66
  ],
67
67
  "title": "ActRequest",
68
68
  "type": "object",
69
- "x-contract-version": "0.5.35"
69
+ "x-contract-version": "0.5.36"
70
70
  }
@@ -75,5 +75,5 @@
75
75
  ],
76
76
  "title": "ActResponse",
77
77
  "type": "object",
78
- "x-contract-version": "0.5.35"
78
+ "x-contract-version": "0.5.36"
79
79
  }
@@ -99,5 +99,5 @@
99
99
  ],
100
100
  "title": "ActionChunk",
101
101
  "type": "object",
102
- "x-contract-version": "0.5.35"
102
+ "x-contract-version": "0.5.36"
103
103
  }
@@ -96,5 +96,5 @@
96
96
  ],
97
97
  "title": "ActionContract",
98
98
  "type": "object",
99
- "x-contract-version": "0.5.35"
99
+ "x-contract-version": "0.5.36"
100
100
  }
@@ -19,5 +19,5 @@
19
19
  ],
20
20
  "title": "ActionStep",
21
21
  "type": "object",
22
- "x-contract-version": "0.5.35"
22
+ "x-contract-version": "0.5.36"
23
23
  }
@@ -17,5 +17,5 @@
17
17
  ],
18
18
  "title": "AdapterPair",
19
19
  "type": "object",
20
- "x-contract-version": "0.5.35"
20
+ "x-contract-version": "0.5.36"
21
21
  }
@@ -23,5 +23,5 @@
23
23
  ],
24
24
  "title": "AddLocal",
25
25
  "type": "object",
26
- "x-contract-version": "0.5.35"
26
+ "x-contract-version": "0.5.36"
27
27
  }
@@ -36,5 +36,5 @@
36
36
  },
37
37
  "title": "Autoscaler",
38
38
  "type": "object",
39
- "x-contract-version": "0.5.35"
39
+ "x-contract-version": "0.5.36"
40
40
  }
@@ -766,5 +766,5 @@
766
766
  ],
767
767
  "title": "BenchmarkRegisterRequest",
768
768
  "type": "object",
769
- "x-contract-version": "0.5.35"
769
+ "x-contract-version": "0.5.36"
770
770
  }
@@ -435,7 +435,7 @@
435
435
  ],
436
436
  "title": "SeqFlags",
437
437
  "type": "integer",
438
- "x-contract-version": "0.5.35",
438
+ "x-contract-version": "0.5.36",
439
439
  "x-flag-names": {
440
440
  "ACTION": 1048576,
441
441
  "CHECK": 65536,
@@ -702,5 +702,5 @@
702
702
  ],
703
703
  "title": "BenchmarkSpec",
704
704
  "type": "object",
705
- "x-contract-version": "0.5.35"
705
+ "x-contract-version": "0.5.36"
706
706
  }
@@ -40,5 +40,5 @@
40
40
  ],
41
41
  "title": "BenchmarkStatusResponse",
42
42
  "type": "object",
43
- "x-contract-version": "0.5.35"
43
+ "x-contract-version": "0.5.36"
44
44
  }
@@ -79,5 +79,5 @@
79
79
  ],
80
80
  "title": "BootstrapResponse",
81
81
  "type": "object",
82
- "x-contract-version": "0.5.35"
82
+ "x-contract-version": "0.5.36"
83
83
  }
@@ -72,5 +72,5 @@
72
72
  ],
73
73
  "title": "Camera",
74
74
  "type": "object",
75
- "x-contract-version": "0.5.35"
75
+ "x-contract-version": "0.5.36"
76
76
  }
@@ -23,5 +23,5 @@
23
23
  ],
24
24
  "title": "CodeBundle",
25
25
  "type": "object",
26
- "x-contract-version": "0.5.35"
26
+ "x-contract-version": "0.5.36"
27
27
  }
@@ -109,5 +109,5 @@
109
109
  },
110
110
  "title": "Concurrency",
111
111
  "type": "object",
112
- "x-contract-version": "0.5.35"
112
+ "x-contract-version": "0.5.36"
113
113
  }
@@ -24,5 +24,5 @@
24
24
  ],
25
25
  "title": "ConnectRequest",
26
26
  "type": "object",
27
- "x-contract-version": "0.5.35"
27
+ "x-contract-version": "0.5.36"
28
28
  }
@@ -200,5 +200,5 @@
200
200
  ],
201
201
  "title": "ConnectResponse",
202
202
  "type": "object",
203
- "x-contract-version": "0.5.35"
203
+ "x-contract-version": "0.5.36"
204
204
  }
@@ -18,5 +18,5 @@
18
18
  ],
19
19
  "title": "DeploymentAccepted",
20
20
  "type": "object",
21
- "x-contract-version": "0.5.35"
21
+ "x-contract-version": "0.5.36"
22
22
  }
@@ -328,5 +328,5 @@
328
328
  ],
329
329
  "title": "DeploymentPolicy",
330
330
  "type": "object",
331
- "x-contract-version": "0.5.35"
331
+ "x-contract-version": "0.5.36"
332
332
  }
@@ -865,8 +865,7 @@
865
865
  }
866
866
  ],
867
867
  "default": null,
868
- "deprecated": true,
869
- "description": "Not accepted: a deployment's workers run wherever there are cards, so a region is refused. Omit it (an older SDK still sends it).",
868
+ "description": "Where its workers run: \"auto\" (the default), each robot's in the region nearest it, or \"us-west\" / \"us-east\", every worker there alone. Any other value is refused.",
870
869
  "title": "Region"
871
870
  }
872
871
  },
@@ -1101,5 +1100,5 @@
1101
1100
  ],
1102
1101
  "title": "DeploymentRequest",
1103
1102
  "type": "object",
1104
- "x-contract-version": "0.5.35"
1103
+ "x-contract-version": "0.5.36"
1105
1104
  }
@@ -58,5 +58,5 @@
58
58
  ],
59
59
  "title": "DeploymentStatusResponse",
60
60
  "type": "object",
61
- "x-contract-version": "0.5.35"
61
+ "x-contract-version": "0.5.36"
62
62
  }
@@ -94,5 +94,5 @@
94
94
  ],
95
95
  "title": "ErrorBody",
96
96
  "type": "object",
97
- "x-contract-version": "0.5.35"
97
+ "x-contract-version": "0.5.36"
98
98
  }
@@ -22,7 +22,7 @@
22
22
  ],
23
23
  "title": "ErrorCode",
24
24
  "type": "string",
25
- "x-contract-version": "0.5.35",
25
+ "x-contract-version": "0.5.36",
26
26
  "x-errors": {
27
27
  "budget_exhausted": {
28
28
  "exit_code": 6,
@@ -60,5 +60,5 @@
60
60
  ],
61
61
  "title": "Estimate",
62
62
  "type": "object",
63
- "x-contract-version": "0.5.35"
63
+ "x-contract-version": "0.5.36"
64
64
  }
@@ -240,5 +240,5 @@
240
240
  ],
241
241
  "title": "EvalDryRunResponse",
242
242
  "type": "object",
243
- "x-contract-version": "0.5.35"
243
+ "x-contract-version": "0.5.36"
244
244
  }
@@ -405,5 +405,5 @@
405
405
  ],
406
406
  "title": "EvalReport",
407
407
  "type": "object",
408
- "x-contract-version": "0.5.35"
408
+ "x-contract-version": "0.5.36"
409
409
  }
@@ -269,5 +269,5 @@
269
269
  },
270
270
  "title": "EvalRequest",
271
271
  "type": "object",
272
- "x-contract-version": "0.5.35"
272
+ "x-contract-version": "0.5.36"
273
273
  }
@@ -153,5 +153,5 @@
153
153
  ],
154
154
  "title": "EvalResponse",
155
155
  "type": "object",
156
- "x-contract-version": "0.5.35"
156
+ "x-contract-version": "0.5.36"
157
157
  }
@@ -49,5 +49,5 @@
49
49
  ],
50
50
  "title": "ImageFrame",
51
51
  "type": "object",
52
- "x-contract-version": "0.5.35"
52
+ "x-contract-version": "0.5.36"
53
53
  }