dinkster-api 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dinkster_api/__init__.py +9 -0
- dinkster_api/py.typed +0 -0
- dinkster_api/v1.py +935 -0
- dinkster_api-0.0.2.dist-info/METADATA +16 -0
- dinkster_api-0.0.2.dist-info/RECORD +238 -0
- dinkster_api-0.0.2.dist-info/WHEEL +4 -0
- dinkster_api-0.0.2.dist-info/entry_points.txt +2 -0
- dinkster_assets/__init__.py +337 -0
- dinkster_assets/acquire.py +226 -0
- dinkster_assets/acquisition_plan.py +145 -0
- dinkster_assets/audio.py +119 -0
- dinkster_assets/catalog.py +156 -0
- dinkster_assets/component_manifest.py +110 -0
- dinkster_assets/declared.py +218 -0
- dinkster_assets/fetch.py +495 -0
- dinkster_assets/guess.py +136 -0
- dinkster_assets/identity.py +72 -0
- dinkster_assets/integrity.py +310 -0
- dinkster_assets/json_metadata.py +41 -0
- dinkster_assets/kind.py +52 -0
- dinkster_assets/latent_format.py +346 -0
- dinkster_assets/library.py +816 -0
- dinkster_assets/media.py +1385 -0
- dinkster_assets/migrations/env.py +38 -0
- dinkster_assets/migrations/script.py.mako +28 -0
- dinkster_assets/migrations/versions/0001_asset_records_create_asset_record_catalog.py +109 -0
- dinkster_assets/migrations/versions/0002_asset_observations.py +36 -0
- dinkster_assets/model.py +242 -0
- dinkster_assets/mounts.py +1278 -0
- dinkster_assets/need.py +168 -0
- dinkster_assets/pack_assets.py +217 -0
- dinkster_assets/probe.py +118 -0
- dinkster_assets/provenance.py +254 -0
- dinkster_assets/public_acquisition.py +424 -0
- dinkster_assets/py.typed +0 -0
- dinkster_assets/records.py +899 -0
- dinkster_assets/resolution.py +778 -0
- dinkster_assets/resolver_index.py +364 -0
- dinkster_assets/resolver_subscription.py +1149 -0
- dinkster_assets/save_target.py +139 -0
- dinkster_assets/schema.py +73 -0
- dinkster_assets/selections.py +256 -0
- dinkster_assets/transport.py +243 -0
- dinkster_assets/value.py +258 -0
- dinkster_assets/vault.py +276 -0
- dinkster_assets/writer.py +765 -0
- dinkster_caches/__init__.py +19 -0
- dinkster_caches/cas.py +310 -0
- dinkster_caches/disk.py +330 -0
- dinkster_caches/layered.py +150 -0
- dinkster_caches/memory.py +208 -0
- dinkster_caches/py.typed +0 -0
- dinkster_caches/wire.py +346 -0
- dinkster_image_document/__init__.py +39 -0
- dinkster_image_document/blend.py +264 -0
- dinkster_image_document/compat.py +211 -0
- dinkster_image_document/document.py +374 -0
- dinkster_image_document/format.py +569 -0
- dinkster_image_document/interchange.py +572 -0
- dinkster_image_document/py.typed +1 -0
- dinkster_image_document/render.py +455 -0
- dinkster_inference_wire/__init__.py +141 -0
- dinkster_inference_wire/camera_wire.py +127 -0
- dinkster_inference_wire/conditioning.py +752 -0
- dinkster_inference_wire/conditioning_wire.py +1143 -0
- dinkster_inference_wire/devices.py +285 -0
- dinkster_inference_wire/generation.py +446 -0
- dinkster_inference_wire/hook_values.py +168 -0
- dinkster_inference_wire/ic_lora_parameters.py +69 -0
- dinkster_inference_wire/latents.py +244 -0
- dinkster_inference_wire/media_wire.py +169 -0
- dinkster_inference_wire/openai_generation.py +1035 -0
- dinkster_inference_wire/pose_keypoint.py +83 -0
- dinkster_inference_wire/process.py +33 -0
- dinkster_inference_wire/py.typed +0 -0
- dinkster_inference_wire/registry.py +116 -0
- dinkster_inference_wire/sam3_wire.py +68 -0
- dinkster_inference_wire/sampling_wire.py +279 -0
- dinkster_inference_wire/sources.py +399 -0
- dinkster_inference_wire/spaces.py +344 -0
- dinkster_inference_wire/sparse.py +372 -0
- dinkster_inference_wire/splat_wire.py +23 -0
- dinkster_inference_wire/types.py +256 -0
- dinkster_inference_wire/vision_wire.py +53 -0
- dinkster_inference_wire/voxel_wire.py +72 -0
- dinkster_inference_wire/wan_wire.py +93 -0
- dinkster_inference_wire/weights.py +219 -0
- dinkster_memory/__init__.py +92 -0
- dinkster_memory/accelerator.py +96 -0
- dinkster_memory/budgets.py +96 -0
- dinkster_memory/details.py +91 -0
- dinkster_memory/governor.py +554 -0
- dinkster_memory/leases.py +176 -0
- dinkster_memory/py.typed +0 -0
- dinkster_memory/release.py +131 -0
- dinkster_memory/reported.py +101 -0
- dinkster_memory/reservations.py +101 -0
- dinkster_memory/resident.py +600 -0
- dinkster_memory/system.py +614 -0
- dinkster_memory/tenants.py +67 -0
- dinkster_protocol/__init__.py +1008 -0
- dinkster_protocol/attention.py +652 -0
- dinkster_protocol/extension_contribution_kinds.json +21 -0
- dinkster_protocol/extension_contribution_kinds.py +43 -0
- dinkster_protocol/extensions.py +742 -0
- dinkster_protocol/frontend_modules.py +145 -0
- dinkster_protocol/multigpu.py +107 -0
- dinkster_protocol/pack_surfaces.py +489 -0
- dinkster_protocol/payload.py +63 -0
- dinkster_protocol/preview.py +86 -0
- dinkster_protocol/py.typed +0 -0
- dinkster_protocol/result_algebra.py +380 -0
- dinkster_protocol/result_document.py +244 -0
- dinkster_protocol/run_control.py +96 -0
- dinkster_protocol/training.py +399 -0
- dinkster_protocol/workgroup.py +1014 -0
- dinkster_registry/__init__.py +135 -0
- dinkster_registry/admission.py +372 -0
- dinkster_registry/artifact.py +128 -0
- dinkster_registry/composition.py +152 -0
- dinkster_registry/grants.py +206 -0
- dinkster_registry/install.py +1017 -0
- dinkster_registry/model.py +414 -0
- dinkster_registry/principals.py +494 -0
- dinkster_registry/py.typed +0 -0
- dinkster_schema/__init__.py +347 -0
- dinkster_schema/capture.py +240 -0
- dinkster_schema/coerce.py +186 -0
- dinkster_schema/comfy_alias.py +496 -0
- dinkster_schema/comfy_group.py +768 -0
- dinkster_schema/elaborate.py +676 -0
- dinkster_schema/execution.py +24 -0
- dinkster_schema/expansion.py +123 -0
- dinkster_schema/log.py +206 -0
- dinkster_schema/media.py +77 -0
- dinkster_schema/model.py +2920 -0
- dinkster_schema/names.py +25 -0
- dinkster_schema/node.py +137 -0
- dinkster_schema/output_descriptors.py +63 -0
- dinkster_schema/py.typed +0 -0
- dinkster_schema/replace.py +1853 -0
- dinkster_schema/reporting.py +441 -0
- dinkster_schema/solve.py +178 -0
- dinkster_schema/wire.py +1876 -0
- dinkster_values/__init__.py +625 -0
- dinkster_values/_media_array.py +7 -0
- dinkster_values/absent.py +77 -0
- dinkster_values/asset_folders.py +64 -0
- dinkster_values/assets.py +106 -0
- dinkster_values/audio_codec.py +594 -0
- dinkster_values/audio_lazy.py +871 -0
- dinkster_values/builtin_types.py +181 -0
- dinkster_values/curve.py +260 -0
- dinkster_values/detection_codec.py +286 -0
- dinkster_values/image_codec.py +691 -0
- dinkster_values/image_components.py +219 -0
- dinkster_values/latent_codec.py +514 -0
- dinkster_values/limits.py +52 -0
- dinkster_values/lists.py +216 -0
- dinkster_values/media_containers.py +70 -0
- dinkster_values/model.py +417 -0
- dinkster_values/model3d_codec.py +167 -0
- dinkster_values/names.py +168 -0
- dinkster_values/pins.py +77 -0
- dinkster_values/py.typed +0 -0
- dinkster_values/registry.py +1105 -0
- dinkster_values/resident.py +210 -0
- dinkster_values/resources.py +210 -0
- dinkster_values/splat_codec.py +477 -0
- dinkster_values/storage.py +133 -0
- dinkster_values/streams.py +241 -0
- dinkster_values/svg_codec.py +120 -0
- dinkster_values/timeline_video.py +68 -0
- dinkster_values/tuples.py +61 -0
- dinkster_values/video_codec.py +897 -0
- dinkster_values/video_document.py +315 -0
- dinkster_values/video_edits.py +253 -0
- dinkster_values/video_probe.py +221 -0
- dinkster_video/__init__.py +58 -0
- dinkster_video/document.py +352 -0
- dinkster_video/formats.py +722 -0
- dinkster_video/image_math.py +175 -0
- dinkster_video/preview/dinkster-pack.toml +36 -0
- dinkster_video/preview/src/dinkster_video_preview/__init__.py +84 -0
- dinkster_video/preview/src/dinkster_video_preview/preview.js +45 -0
- dinkster_video/py.typed +1 -0
- dinkster_video/runtime.py +1561 -0
- dinkster_video/timeline.py +576 -0
- dinkster_video/timeline_runtime.py +305 -0
- dinkster_video_preview/__init__.py +84 -0
- dinkster_video_preview/preview.js +45 -0
- dinkster_workers/__init__.py +207 -0
- dinkster_workers/_doctor_probe.py +359 -0
- dinkster_workers/accelerator.py +156 -0
- dinkster_workers/aimdo_bootstrap.py +81 -0
- dinkster_workers/backend_env.py +1485 -0
- dinkster_workers/blobs.py +402 -0
- dinkster_workers/boundary.py +1918 -0
- dinkster_workers/catalog.py +182 -0
- dinkster_workers/devices.py +164 -0
- dinkster_workers/diagnostics.py +64 -0
- dinkster_workers/dispatch.py +298 -0
- dinkster_workers/doctor.py +1382 -0
- dinkster_workers/egress.py +344 -0
- dinkster_workers/error_hints.py +175 -0
- dinkster_workers/execution.py +160 -0
- dinkster_workers/governed.py +123 -0
- dinkster_workers/headroom.py +156 -0
- dinkster_workers/host.py +3116 -0
- dinkster_workers/in_process.py +1325 -0
- dinkster_workers/interpreter.py +84 -0
- dinkster_workers/isolated.py +862 -0
- dinkster_workers/launch.py +72 -0
- dinkster_workers/manifest.py +3332 -0
- dinkster_workers/media.py +196 -0
- dinkster_workers/pack_environment.py +473 -0
- dinkster_workers/pack_lock.py +359 -0
- dinkster_workers/placement.py +215 -0
- dinkster_workers/preview_artifacts.py +233 -0
- dinkster_workers/produced_assets.py +218 -0
- dinkster_workers/provision.py +597 -0
- dinkster_workers/py.typed +0 -0
- dinkster_workers/relay.py +762 -0
- dinkster_workers/remote.py +921 -0
- dinkster_workers/resume.py +490 -0
- dinkster_workers/routing.py +134 -0
- dinkster_workers/run_control.py +275 -0
- dinkster_workers/sandbox.py +798 -0
- dinkster_workers/saved_artifacts.py +275 -0
- dinkster_workers/service.py +593 -0
- dinkster_workers/session.py +2605 -0
- dinkster_workers/single_job_multi_gpu.py +7 -0
- dinkster_workers/staging.py +230 -0
- dinkster_workers/transport.py +161 -0
- dinkster_workers/workgroup.py +729 -0
- dinkster_workers/workgroup_compose.py +135 -0
- dinkster_workers/workgroup_config.py +196 -0
- dinkster_workers/workgroup_session.py +340 -0
|
@@ -0,0 +1,1918 @@
|
|
|
1
|
+
"""The process boundary: framed messages and the value wire codec.
|
|
2
|
+
|
|
3
|
+
Everything that crosses between an engine process and an isolated worker
|
|
4
|
+
process goes through here, in both directions:
|
|
5
|
+
|
|
6
|
+
- Frames: ``uint32 header length | JSON header | binary blobs``. The header
|
|
7
|
+
carries structure (message type, ids, value descriptors, schema wire); the
|
|
8
|
+
blobs carry payload and meta bytes so binary data never rides base64.
|
|
9
|
+
- Values cross as envelopes (hazard H2): type id, fingerprint, and meta in
|
|
10
|
+
the header; payload bytes produced by the type's registered codec. The
|
|
11
|
+
fingerprint travels verbatim, so cache keys stay location-independent
|
|
12
|
+
(hazard H4) - a value fingerprinted in a worker keys the same cache entry
|
|
13
|
+
everywhere.
|
|
14
|
+
- Payload transports (DESIGN 3.2): ``inline`` (bytes in the frame) and
|
|
15
|
+
``shm`` (bytes in a POSIX shared-memory segment, for encoded payloads at
|
|
16
|
+
or above a size threshold). Which transport carries a value is a placement
|
|
17
|
+
decision made here; node code never sees it (hazard H9).
|
|
18
|
+
|
|
19
|
+
shm segments are single-hop handoffs: the sender creates and fills a
|
|
20
|
+
segment (keeping its handle open - see ``release_segment``), the receiver
|
|
21
|
+
retains a read-only mapping and acknowledges (``shmAck``), and the sender
|
|
22
|
+
unlinks its name. Anything unacknowledged when a side shuts down is released
|
|
23
|
+
by its creator (hazard H14).
|
|
24
|
+
"""
|
|
25
|
+
# pyright: reportUnknownArgumentType=false, reportUnknownMemberType=false
|
|
26
|
+
# pyright: reportUnknownVariableType=false, reportUnnecessaryIsInstance=false
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import asyncio
|
|
31
|
+
import contextlib
|
|
32
|
+
import json
|
|
33
|
+
import math
|
|
34
|
+
import mmap
|
|
35
|
+
import os
|
|
36
|
+
import sys
|
|
37
|
+
import time
|
|
38
|
+
import uuid
|
|
39
|
+
from collections.abc import Callable, Collection, Generator, Mapping, Sequence
|
|
40
|
+
from dataclasses import dataclass
|
|
41
|
+
from multiprocessing.shared_memory import SharedMemory
|
|
42
|
+
from pathlib import Path
|
|
43
|
+
from typing import Any, Protocol, cast
|
|
44
|
+
|
|
45
|
+
from dinkster_assets import AssetError, digest_bytes
|
|
46
|
+
from dinkster_protocol import (
|
|
47
|
+
AssetFolderContext,
|
|
48
|
+
ErrorHint,
|
|
49
|
+
ExportSnapshot,
|
|
50
|
+
Invocation,
|
|
51
|
+
InvocationOutcome,
|
|
52
|
+
InvocationResult,
|
|
53
|
+
MediaSourceAuthority,
|
|
54
|
+
NodeError,
|
|
55
|
+
SavedArtifactCandidate,
|
|
56
|
+
is_extension_snapshot_digest,
|
|
57
|
+
validate_preview_animation,
|
|
58
|
+
validate_preview_mode,
|
|
59
|
+
)
|
|
60
|
+
from dinkster_schema import schema_from_wire, schema_to_wire
|
|
61
|
+
from dinkster_schema.reporting import EncodedPreviewArtifact
|
|
62
|
+
from dinkster_values import (
|
|
63
|
+
CONVERSATION_CAS_BUDGET_BYTES,
|
|
64
|
+
WORKER_FRAME_HEADER_LIMIT_BYTES,
|
|
65
|
+
BufferEncoding,
|
|
66
|
+
EncodedPayload,
|
|
67
|
+
ListPayload,
|
|
68
|
+
TuplePayload,
|
|
69
|
+
TypeRegistry,
|
|
70
|
+
Value,
|
|
71
|
+
ValueMeta,
|
|
72
|
+
default_decode,
|
|
73
|
+
default_encode,
|
|
74
|
+
parse_list_type_id,
|
|
75
|
+
parse_tuple_type_id,
|
|
76
|
+
stable_hash,
|
|
77
|
+
value_children,
|
|
78
|
+
)
|
|
79
|
+
from dinkster_values.storage import encoded_storage_meta
|
|
80
|
+
|
|
81
|
+
#: Windows and macOS report shared-memory mappings rounded up to a page
|
|
82
|
+
#: multiple (macOS ftruncates POSIX shm to 16 KiB pages on Apple Silicon);
|
|
83
|
+
#: Linux reports the logical size exactly.
|
|
84
|
+
_SHM_MAPPING_ROUNDS = os.name == "nt" or sys.platform == "darwin"
|
|
85
|
+
|
|
86
|
+
DEFAULT_SHM_THRESHOLD = 256 * 1024
|
|
87
|
+
"""Encoded payloads at or above this many bytes ride shared memory instead
|
|
88
|
+
of the socket frame."""
|
|
89
|
+
|
|
90
|
+
DEFAULT_CAS_THRESHOLD = 256 * 1024
|
|
91
|
+
"""Encoded payloads at or above this many bytes are content-addressed over
|
|
92
|
+
a network boundary: repeat crossings of the same bytes send a digest-only
|
|
93
|
+
descriptor instead of the payload."""
|
|
94
|
+
|
|
95
|
+
DEFAULT_CAS_BUDGET = CONVERSATION_CAS_BUDGET_BYTES
|
|
96
|
+
"""Cap on payload bytes each side retains for conversation-scoped dedup.
|
|
97
|
+
Once a codec's retained bytes reach this budget, further first-crossings
|
|
98
|
+
still work - they just carry their bytes every time instead of becoming
|
|
99
|
+
digest-only. Bounds the RAM a long-lived connection can pin."""
|
|
100
|
+
|
|
101
|
+
PROTOCOL_VERSION = 12
|
|
102
|
+
"""The boundary protocol version. Carried in the remote hello exchange
|
|
103
|
+
(clientHello/hello) so peers built against different wire contracts refuse
|
|
104
|
+
each other explicitly instead of corrupting frames. Local parent/child pairs
|
|
105
|
+
ship together and do not negotiate.
|
|
106
|
+
|
|
107
|
+
v12: explicit mounted saved-file attachments in successful invocation results;
|
|
108
|
+
v11: encoded preview artifacts in successful invocation results;
|
|
109
|
+
v10: invocation-scoped run-control relay;
|
|
110
|
+
v9: schema declarations and invocation effective schemas;
|
|
111
|
+
v8: stable engine, daemon, job-attempt, and invocation identities with
|
|
112
|
+
same-process rebind, owner fencing, and acknowledged result replay;
|
|
113
|
+
v7: session leases (``engine`` label in clientHello, ``leaseTtl`` in the
|
|
114
|
+
service hello, ``heartbeat``/``heartbeatAck`` frames);
|
|
115
|
+
v6: resident type ids ``dinkster.clip``/``dinkster.vae`` (renamed from
|
|
116
|
+
``dinkster.text_encoder``/``dinkster.codec``);
|
|
117
|
+
v5: role-labeled multi-stream payloads in ``dinkster.latent``;
|
|
118
|
+
v4: saved output artifacts in invocation results;
|
|
119
|
+
v3: lazy-status hook frames and schema wire v16;
|
|
120
|
+
v2: recursive list value descriptors (``elements``) and schema wire v2
|
|
121
|
+
(recursive ``element`` type expressions)."""
|
|
122
|
+
|
|
123
|
+
_MAX_HEADER_BYTES = WORKER_FRAME_HEADER_LIMIT_BYTES
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class BoundaryError(Exception):
|
|
127
|
+
"""A frame or value could not cross the process boundary."""
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class EncodedPayloadValidationError(BoundaryError):
|
|
131
|
+
"""A registered type refused encoded bytes before their fingerprint was trusted."""
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class PersistentBlobMissing(BoundaryError):
|
|
135
|
+
"""A persistentCas descriptor named a digest the local value store cannot resolve."""
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class ValueStore(Protocol):
|
|
139
|
+
"""The persistent blob store behind the ``persistentCas`` transport,
|
|
140
|
+
on either side of the boundary. dinkster-caches' BudgetedDiskCAS (and
|
|
141
|
+
DiskCAS) satisfy it structurally. Synchronous like DiskCAS: async
|
|
142
|
+
callers wrap calls in ``asyncio.to_thread``."""
|
|
143
|
+
|
|
144
|
+
@property
|
|
145
|
+
def root(self) -> Path: ...
|
|
146
|
+
|
|
147
|
+
def has(self, digest: str) -> bool: ...
|
|
148
|
+
|
|
149
|
+
def resolve(self, digest: str) -> Path | None: ...
|
|
150
|
+
|
|
151
|
+
def get(self, digest: str) -> bytes | None: ...
|
|
152
|
+
|
|
153
|
+
def put(self, data: bytes, *, protect: Collection[str] = ()) -> str: ...
|
|
154
|
+
|
|
155
|
+
def adopt_file(self, path: Path, digest: str, *, protect: Collection[str] = ()) -> None: ...
|
|
156
|
+
|
|
157
|
+
def pin(self, owner: object, digests: Collection[str]) -> None: ...
|
|
158
|
+
|
|
159
|
+
def unpin(self, owner: object) -> None: ...
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
@dataclass(frozen=True)
|
|
163
|
+
class TransferStat:
|
|
164
|
+
"""What one value's boundary crossing cost on this side.
|
|
165
|
+
|
|
166
|
+
``codec_ms`` is time spent encoding (sender) or attaching shared memory
|
|
167
|
+
(receiver) at transfer time; lazy ``load()`` decode is not included.
|
|
168
|
+
``reused`` means encoded bytes were relayed without re-encoding.
|
|
169
|
+
``network_bytes``/``transfer_ms`` are payload bytes the persistentCas
|
|
170
|
+
transport actually streamed for this edge and the time spent streaming
|
|
171
|
+
them - zero on a store hit, and zero for the other transports (their
|
|
172
|
+
movement is implied: inline and first-crossing cas bytes ride the
|
|
173
|
+
frame itself)."""
|
|
174
|
+
|
|
175
|
+
transport: str
|
|
176
|
+
size_bytes: int
|
|
177
|
+
codec_ms: float
|
|
178
|
+
declared_codec: bool
|
|
179
|
+
reused: bool
|
|
180
|
+
network_bytes: int = 0
|
|
181
|
+
transfer_ms: float = 0.0
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
async def read_frame(
|
|
185
|
+
reader: asyncio.StreamReader,
|
|
186
|
+
) -> tuple[dict[str, Any], list[bytes]] | None:
|
|
187
|
+
"""Read one frame. Returns None on clean or broken EOF."""
|
|
188
|
+
try:
|
|
189
|
+
raw = await reader.readexactly(4)
|
|
190
|
+
except (asyncio.IncompleteReadError, ConnectionError):
|
|
191
|
+
return None
|
|
192
|
+
size = int.from_bytes(raw, "big")
|
|
193
|
+
if size > _MAX_HEADER_BYTES:
|
|
194
|
+
raise BoundaryError(f"frame header too large: {size} bytes")
|
|
195
|
+
try:
|
|
196
|
+
header_obj = cast(
|
|
197
|
+
object,
|
|
198
|
+
json.loads(
|
|
199
|
+
await reader.readexactly(size),
|
|
200
|
+
object_pairs_hook=_unique_object,
|
|
201
|
+
parse_constant=lambda value: (_ for _ in ()).throw(ValueError(value)),
|
|
202
|
+
),
|
|
203
|
+
)
|
|
204
|
+
if not isinstance(header_obj, dict):
|
|
205
|
+
raise BoundaryError("frame header must be a JSON object")
|
|
206
|
+
header = cast("dict[str, Any]", header_obj)
|
|
207
|
+
blobs: list[bytes] = []
|
|
208
|
+
for length in header.get("blobs", ()):
|
|
209
|
+
if type(length) is not int or length < 0:
|
|
210
|
+
raise BoundaryError("frame blob lengths must be non-negative integers")
|
|
211
|
+
blobs.append(await reader.readexactly(length))
|
|
212
|
+
except (asyncio.IncompleteReadError, ConnectionError):
|
|
213
|
+
return None
|
|
214
|
+
except (ValueError, TypeError, json.JSONDecodeError, RecursionError, UnicodeError) as exc:
|
|
215
|
+
raise BoundaryError("frame header is not strict JSON") from exc
|
|
216
|
+
return header, blobs
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
async def write_frame(
|
|
220
|
+
writer: asyncio.StreamWriter, header: Mapping[str, object], blobs: Sequence[bytes]
|
|
221
|
+
) -> None:
|
|
222
|
+
"""Write one frame. Callers serialize access (frames must not interleave)."""
|
|
223
|
+
full = dict(header)
|
|
224
|
+
full["blobs"] = [len(blob) for blob in blobs]
|
|
225
|
+
try:
|
|
226
|
+
data = json.dumps(full, separators=(",", ":"), allow_nan=False).encode("utf-8")
|
|
227
|
+
except (ValueError, TypeError, RecursionError, UnicodeError) as exc:
|
|
228
|
+
raise BoundaryError("frame header is not strict JSON") from exc
|
|
229
|
+
if len(data) > _MAX_HEADER_BYTES:
|
|
230
|
+
raise BoundaryError(f"frame header too large: {len(data)} bytes")
|
|
231
|
+
writer.write(len(data).to_bytes(4, "big"))
|
|
232
|
+
writer.write(data)
|
|
233
|
+
for blob in blobs:
|
|
234
|
+
writer.write(blob)
|
|
235
|
+
await writer.drain()
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _create_empty_segment(size: int) -> SharedMemory:
|
|
239
|
+
return SharedMemory(name="dinkster" + uuid.uuid4().hex[:16], create=True, size=size)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _create_segment(data: bytes | memoryview) -> SharedMemory:
|
|
243
|
+
segment = _create_empty_segment(len(data))
|
|
244
|
+
buffer = segment.buf
|
|
245
|
+
assert buffer is not None # always set on a live segment
|
|
246
|
+
try:
|
|
247
|
+
buffer[: len(data)] = data
|
|
248
|
+
except Exception:
|
|
249
|
+
buffer.release()
|
|
250
|
+
release_segment(segment)
|
|
251
|
+
raise
|
|
252
|
+
return segment
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _create_segment_from_encoding(encoding: BufferEncoding) -> SharedMemory:
|
|
256
|
+
segment = _create_empty_segment(encoding.size)
|
|
257
|
+
buffer = segment.buf
|
|
258
|
+
assert buffer is not None # always set on a live segment
|
|
259
|
+
target = buffer[: encoding.size]
|
|
260
|
+
try:
|
|
261
|
+
written = encoding.write(target)
|
|
262
|
+
if written != encoding.size:
|
|
263
|
+
raise ValueError(f"buffer encoder wrote {written} bytes, expected {encoding.size}")
|
|
264
|
+
except Exception:
|
|
265
|
+
with contextlib.suppress(Exception):
|
|
266
|
+
target.release()
|
|
267
|
+
with contextlib.suppress(Exception):
|
|
268
|
+
buffer.release()
|
|
269
|
+
release_segment(segment)
|
|
270
|
+
raise
|
|
271
|
+
target.release()
|
|
272
|
+
return segment
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _attach_segment(name: str) -> SharedMemory:
|
|
276
|
+
if sys.version_info >= (3, 13):
|
|
277
|
+
return SharedMemory(name=name, track=False)
|
|
278
|
+
segment = SharedMemory(name=name)
|
|
279
|
+
if os.name == "posix":
|
|
280
|
+
# Pre-3.13 CPython registers even attach-only handles with the
|
|
281
|
+
# resource tracker, which would unlink the segment when *this*
|
|
282
|
+
# process exits - but the creator owns unlinking (hazard H14).
|
|
283
|
+
from multiprocessing import resource_tracker
|
|
284
|
+
|
|
285
|
+
resource_tracker.unregister(f"/{name}", "shared_memory")
|
|
286
|
+
return segment
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def release_segment(segment: SharedMemory) -> None:
|
|
290
|
+
"""Drop a sent segment after acknowledgement (or on shutdown): close our
|
|
291
|
+
handle, then unlink the name.
|
|
292
|
+
|
|
293
|
+
The sender must hold its handle open from send until ack - on Windows a
|
|
294
|
+
segment is freed when its last handle closes, so closing at send time
|
|
295
|
+
would destroy it before the receiver attaches. Close-then-unlink is the
|
|
296
|
+
portable release order (unlink is a no-op on Windows)."""
|
|
297
|
+
with contextlib.suppress(Exception):
|
|
298
|
+
segment.close()
|
|
299
|
+
with contextlib.suppress(Exception):
|
|
300
|
+
segment.unlink()
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _open_segment(name: str, size: int) -> tuple[SharedMemory, memoryview]:
|
|
304
|
+
if size <= 0:
|
|
305
|
+
raise BoundaryError("shared-memory payload size must be positive")
|
|
306
|
+
try:
|
|
307
|
+
segment = _attach_segment(name)
|
|
308
|
+
except FileNotFoundError as exc:
|
|
309
|
+
raise BoundaryError(f"shared-memory segment '{name}' has vanished") from exc
|
|
310
|
+
buffer = segment.buf
|
|
311
|
+
assert buffer is not None # always set on a live segment
|
|
312
|
+
actual_size = len(buffer)
|
|
313
|
+
expected_size = (
|
|
314
|
+
((size + mmap.PAGESIZE - 1) // mmap.PAGESIZE) * mmap.PAGESIZE
|
|
315
|
+
if _SHM_MAPPING_ROUNDS
|
|
316
|
+
else size
|
|
317
|
+
)
|
|
318
|
+
if actual_size != expected_size:
|
|
319
|
+
segment.close()
|
|
320
|
+
raise BoundaryError(
|
|
321
|
+
f"shared-memory segment '{name}' has size {actual_size}, expected {expected_size}"
|
|
322
|
+
)
|
|
323
|
+
return segment, buffer
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
class ValueCodec:
|
|
327
|
+
"""Encodes/decodes Values for one side of a boundary.
|
|
328
|
+
|
|
329
|
+
Encoding uses the type's registered codec; values that arrived encoded
|
|
330
|
+
(EncodedPayload) relay their bytes as-is, so unknown types can flow
|
|
331
|
+
*through* a process that cannot decode them. The transport choice
|
|
332
|
+
(inline vs shm vs cas) is made per value from the encoded size.
|
|
333
|
+
|
|
334
|
+
The ``cas`` transport is conversation-scoped content addressing for
|
|
335
|
+
network boundaries (where shm is structurally refused): the first
|
|
336
|
+
crossing of a payload carries digest + bytes, and every later crossing
|
|
337
|
+
of the same bytes - in either direction - carries the digest alone.
|
|
338
|
+
Both sides retain each crossed payload in ``_conversation`` (keyed by
|
|
339
|
+
digest), so a digest-only descriptor always resolves locally; the store
|
|
340
|
+
lives and dies with the codec, i.e. with the conversation (hazard H14:
|
|
341
|
+
nothing outlives the conversation). Digests here are ``stable_hash``
|
|
342
|
+
(stdlib blake2b) - conversation-local dedup identity, deliberately not
|
|
343
|
+
the canonical ``blake3:`` CAS namespace, so isolated pack venvs never
|
|
344
|
+
need the blake3 wheel. Enabled only after hello negotiation
|
|
345
|
+
(``enable_cas``); never assumed.
|
|
346
|
+
|
|
347
|
+
Retention is bounded (``cas_budget``): the *sender* decides whether a
|
|
348
|
+
first-crossing payload is retained (fits its remaining budget) and says
|
|
349
|
+
so in the descriptor (``retained``); the receiver mirrors that decision
|
|
350
|
+
verbatim. Sender-decides-receiver-mirrors keeps the two stores
|
|
351
|
+
consistent by construction - a digest-only descriptor only ever names a
|
|
352
|
+
digest both sides kept - so an over-budget conversation degrades to
|
|
353
|
+
resending bytes, never to a protocol error. Nothing is evicted:
|
|
354
|
+
coordinated eviction would need protocol round-trips for RAM that the
|
|
355
|
+
budget already bounds.
|
|
356
|
+
"""
|
|
357
|
+
|
|
358
|
+
def __init__(
|
|
359
|
+
self,
|
|
360
|
+
registry: TypeRegistry,
|
|
361
|
+
*,
|
|
362
|
+
shm_threshold: int = DEFAULT_SHM_THRESHOLD,
|
|
363
|
+
use_shm: bool = True,
|
|
364
|
+
accept_shm: bool = True,
|
|
365
|
+
cas_threshold: int = DEFAULT_CAS_THRESHOLD,
|
|
366
|
+
cas_budget: int = DEFAULT_CAS_BUDGET,
|
|
367
|
+
value_store: ValueStore | None = None,
|
|
368
|
+
) -> None:
|
|
369
|
+
if shm_threshold < 1:
|
|
370
|
+
raise ValueError("shm_threshold must be >= 1")
|
|
371
|
+
if cas_threshold < 1:
|
|
372
|
+
raise ValueError("cas_threshold must be >= 1")
|
|
373
|
+
if cas_budget < 0:
|
|
374
|
+
raise ValueError("cas_budget must be >= 0")
|
|
375
|
+
self._registry = registry
|
|
376
|
+
self._shm_threshold = shm_threshold
|
|
377
|
+
self._use_shm = use_shm
|
|
378
|
+
# A remote peer's segment *name* would attach to whatever local
|
|
379
|
+
# memory happens to share it - never the peer's bytes. A codec for a
|
|
380
|
+
# network boundary refuses shm descriptors outright (accept_shm
|
|
381
|
+
# False) instead of reading garbage that parses.
|
|
382
|
+
self._accept_shm = accept_shm
|
|
383
|
+
self._cas_threshold = cas_threshold
|
|
384
|
+
self._cas_budget = cas_budget
|
|
385
|
+
self._cas = False
|
|
386
|
+
self._conversation: dict[str, bytes] = {}
|
|
387
|
+
self._conversation_bytes = 0
|
|
388
|
+
self._value_store = value_store
|
|
389
|
+
self._persistent = False
|
|
390
|
+
self._pending_store: list[tuple[str, bytes]] = []
|
|
391
|
+
|
|
392
|
+
def _retain(self, digest: str, data: bytes) -> None:
|
|
393
|
+
if digest not in self._conversation:
|
|
394
|
+
self._conversation[digest] = data
|
|
395
|
+
self._conversation_bytes += len(data)
|
|
396
|
+
|
|
397
|
+
def enable_cas(self) -> None:
|
|
398
|
+
"""Turn on the cas transport - called only after both hellos listed
|
|
399
|
+
it, so a descriptor never reaches a peer that cannot resolve it."""
|
|
400
|
+
self._cas = True
|
|
401
|
+
|
|
402
|
+
def enable_persistent_cas(self) -> None:
|
|
403
|
+
"""Turn on the persistentCas transport - called only after both
|
|
404
|
+
hellos listed it AND this side holds a value store. Takes priority
|
|
405
|
+
over the conversation cas transport at encode time, so a persistent
|
|
406
|
+
conversation never populates the conversation-scoped index."""
|
|
407
|
+
if self._value_store is None:
|
|
408
|
+
raise ValueError("persistent cas requires a value store")
|
|
409
|
+
self._persistent = True
|
|
410
|
+
|
|
411
|
+
@contextlib.contextmanager
|
|
412
|
+
def suspend_persistent_cas(self) -> Generator[None]:
|
|
413
|
+
"""Encode without persistentCas descriptors. Result algebra
|
|
414
|
+
documents must be self-contained: they have no blob-transfer
|
|
415
|
+
companion frames, so their values fall back to inline/cas."""
|
|
416
|
+
previous, self._persistent = self._persistent, False
|
|
417
|
+
try:
|
|
418
|
+
yield
|
|
419
|
+
finally:
|
|
420
|
+
self._persistent = previous
|
|
421
|
+
|
|
422
|
+
def take_pending_store_blobs(self) -> list[tuple[str, bytes]]:
|
|
423
|
+
"""Drain the (digest, bytes) pairs the encodes since the last drain
|
|
424
|
+
promised as persistentCas references. The caller must make them
|
|
425
|
+
resolvable in the peer's store (BlobTransfer.ensure_peer_holds)
|
|
426
|
+
before the frame that references them is sent, or drop them when
|
|
427
|
+
the frame is abandoned. Callers drain synchronously after encoding
|
|
428
|
+
(no await in between), so concurrent encoders on one event loop
|
|
429
|
+
never see each other's pending blobs."""
|
|
430
|
+
pending, self._pending_store = self._pending_store, []
|
|
431
|
+
return pending
|
|
432
|
+
|
|
433
|
+
def conversation_checkpoint(self) -> tuple[dict[str, bytes], int]:
|
|
434
|
+
"""Snapshot CAS retention so a multi-value frame can encode transactionally."""
|
|
435
|
+
return dict(self._conversation), self._conversation_bytes
|
|
436
|
+
|
|
437
|
+
def restore_conversation(self, checkpoint: tuple[dict[str, bytes], int]) -> None:
|
|
438
|
+
"""Roll back CAS retention after a frame that was never sent."""
|
|
439
|
+
self._conversation, self._conversation_bytes = checkpoint
|
|
440
|
+
|
|
441
|
+
def encode(
|
|
442
|
+
self, value: Value, blobs: list[bytes], segments: list[SharedMemory]
|
|
443
|
+
) -> tuple[dict[str, object], TransferStat]:
|
|
444
|
+
children = value_children(value)
|
|
445
|
+
if children is not None:
|
|
446
|
+
# Containers cross as recursive descriptors: each child is a full
|
|
447
|
+
# value descriptor with its own payload transport, so child
|
|
448
|
+
# envelopes (resource stubs, residency, cost) survive the
|
|
449
|
+
# boundary exactly like top-level values (DESIGN 3.13).
|
|
450
|
+
element_wires: list[dict[str, object]] = []
|
|
451
|
+
total_size = 0
|
|
452
|
+
total_ms = 0.0
|
|
453
|
+
all_declared = True
|
|
454
|
+
all_reused = bool(children)
|
|
455
|
+
for child in children:
|
|
456
|
+
child_wire, child_stat = self.encode(child, blobs, segments)
|
|
457
|
+
element_wires.append(child_wire)
|
|
458
|
+
total_size += child_stat.size_bytes
|
|
459
|
+
total_ms += child_stat.codec_ms
|
|
460
|
+
all_declared = all_declared and child_stat.declared_codec
|
|
461
|
+
all_reused = all_reused and child_stat.reused
|
|
462
|
+
wire: dict[str, object] = {
|
|
463
|
+
"typeId": value.type_id,
|
|
464
|
+
"fingerprint": value.fingerprint,
|
|
465
|
+
"metaBlob": len(blobs),
|
|
466
|
+
"elements": element_wires,
|
|
467
|
+
}
|
|
468
|
+
blobs.append(default_encode(dict(value.meta.entries)))
|
|
469
|
+
return wire, TransferStat(
|
|
470
|
+
value.payload.transport, total_size, total_ms, all_declared, all_reused
|
|
471
|
+
)
|
|
472
|
+
started = time.perf_counter()
|
|
473
|
+
payload = value.payload
|
|
474
|
+
reused = isinstance(payload, EncodedPayload) and payload.type_id == value.type_id
|
|
475
|
+
if reused:
|
|
476
|
+
encoded = cast(EncodedPayload, payload)
|
|
477
|
+
if self._use_shm and self._shm_threshold <= encoded.size:
|
|
478
|
+
codec_ms = (time.perf_counter() - started) * 1000.0
|
|
479
|
+
meta_data = default_encode(dict(value.meta.entries))
|
|
480
|
+
with encoded.borrow_data() as data:
|
|
481
|
+
segment = _create_segment(data)
|
|
482
|
+
segments.append(segment)
|
|
483
|
+
declared = (
|
|
484
|
+
self._registry.spec(value.type_id).declared_codec
|
|
485
|
+
if value.type_id in self._registry
|
|
486
|
+
else False
|
|
487
|
+
)
|
|
488
|
+
wire = {
|
|
489
|
+
"typeId": value.type_id,
|
|
490
|
+
"fingerprint": value.fingerprint,
|
|
491
|
+
"metaBlob": len(blobs),
|
|
492
|
+
"payload": {
|
|
493
|
+
"transport": "shm",
|
|
494
|
+
"segment": segment.name,
|
|
495
|
+
"size": encoded.size,
|
|
496
|
+
},
|
|
497
|
+
}
|
|
498
|
+
blobs.append(meta_data)
|
|
499
|
+
return wire, TransferStat("shm", encoded.size, codec_ms, declared, True)
|
|
500
|
+
data = encoded.data
|
|
501
|
+
else:
|
|
502
|
+
try:
|
|
503
|
+
spec = self._registry.spec(value.type_id)
|
|
504
|
+
except KeyError as exc:
|
|
505
|
+
raise BoundaryError(
|
|
506
|
+
f"cannot send value of type '{value.type_id}': the type is not "
|
|
507
|
+
"registered in this process and the payload is not already encoded"
|
|
508
|
+
) from exc
|
|
509
|
+
obj = payload.load()
|
|
510
|
+
if self._use_shm and spec.prepare_buffer_encoding is not None:
|
|
511
|
+
encoding = spec.prepare_buffer_encoding(obj)
|
|
512
|
+
if self._shm_threshold <= encoding.size:
|
|
513
|
+
meta_data = default_encode(dict(value.meta.entries))
|
|
514
|
+
segment = _create_segment_from_encoding(encoding)
|
|
515
|
+
segments.append(segment)
|
|
516
|
+
codec_ms = (time.perf_counter() - started) * 1000.0
|
|
517
|
+
wire = {
|
|
518
|
+
"typeId": value.type_id,
|
|
519
|
+
"fingerprint": value.fingerprint,
|
|
520
|
+
"metaBlob": len(blobs),
|
|
521
|
+
"payload": {
|
|
522
|
+
"transport": "shm",
|
|
523
|
+
"segment": segment.name,
|
|
524
|
+
"size": encoding.size,
|
|
525
|
+
},
|
|
526
|
+
}
|
|
527
|
+
blobs.append(meta_data)
|
|
528
|
+
return wire, TransferStat(
|
|
529
|
+
"shm", encoding.size, codec_ms, spec.declared_codec, False
|
|
530
|
+
)
|
|
531
|
+
data = spec.encode(obj)
|
|
532
|
+
codec_ms = (time.perf_counter() - started) * 1000.0
|
|
533
|
+
declared = (
|
|
534
|
+
self._registry.spec(value.type_id).declared_codec
|
|
535
|
+
if value.type_id in self._registry
|
|
536
|
+
else False
|
|
537
|
+
)
|
|
538
|
+
wire: dict[str, object] = {
|
|
539
|
+
"typeId": value.type_id,
|
|
540
|
+
"fingerprint": value.fingerprint,
|
|
541
|
+
"metaBlob": len(blobs),
|
|
542
|
+
}
|
|
543
|
+
blobs.append(default_encode(dict(value.meta.entries)))
|
|
544
|
+
if self._use_shm and 0 < self._shm_threshold <= len(data):
|
|
545
|
+
segment = _create_segment(data)
|
|
546
|
+
segments.append(segment)
|
|
547
|
+
wire["payload"] = {"transport": "shm", "segment": segment.name, "size": len(data)}
|
|
548
|
+
transport = "shm"
|
|
549
|
+
elif self._persistent and self._cas_threshold <= len(data):
|
|
550
|
+
# The frame carries only the digest; the actual bytes are the
|
|
551
|
+
# frame sender's obligation (take_pending_store_blobs) to land
|
|
552
|
+
# in the peer's store before this frame crosses.
|
|
553
|
+
digest = digest_bytes(data)
|
|
554
|
+
self._pending_store.append((digest, data))
|
|
555
|
+
wire["payload"] = {
|
|
556
|
+
"transport": "persistentCas",
|
|
557
|
+
"digest": digest,
|
|
558
|
+
"size": len(data),
|
|
559
|
+
}
|
|
560
|
+
transport = "persistentCas"
|
|
561
|
+
elif self._cas and self._cas_threshold <= len(data):
|
|
562
|
+
digest = stable_hash([data])
|
|
563
|
+
payload_wire: dict[str, object] = {
|
|
564
|
+
"transport": "cas",
|
|
565
|
+
"digest": digest,
|
|
566
|
+
"size": len(data),
|
|
567
|
+
}
|
|
568
|
+
if digest not in self._conversation:
|
|
569
|
+
# First crossing: the bytes ride along. The sender decides
|
|
570
|
+
# whether both sides retain them (fits the budget) and the
|
|
571
|
+
# receiver mirrors that decision, so the stores never
|
|
572
|
+
# disagree about which digests resolve.
|
|
573
|
+
retained = self._conversation_bytes + len(data) <= self._cas_budget
|
|
574
|
+
payload_wire["blob"] = len(blobs)
|
|
575
|
+
payload_wire["retained"] = retained
|
|
576
|
+
blobs.append(data)
|
|
577
|
+
if retained:
|
|
578
|
+
self._retain(digest, data)
|
|
579
|
+
wire["payload"] = payload_wire
|
|
580
|
+
transport = "cas"
|
|
581
|
+
else:
|
|
582
|
+
wire["payload"] = {"transport": "inline", "blob": len(blobs)}
|
|
583
|
+
blobs.append(data)
|
|
584
|
+
transport = "inline"
|
|
585
|
+
return wire, TransferStat(transport, len(data), codec_ms, declared, reused)
|
|
586
|
+
|
|
587
|
+
def decode(
|
|
588
|
+
self,
|
|
589
|
+
wire: Mapping[str, Any],
|
|
590
|
+
blobs: Sequence[bytes],
|
|
591
|
+
consumed_segments: list[str],
|
|
592
|
+
) -> tuple[Value, TransferStat]:
|
|
593
|
+
"""Decode one value descriptor. Shared-memory payloads retain a
|
|
594
|
+
read-only receiver mapping while their segment names are acknowledged;
|
|
595
|
+
object decode and owned-byte materialization stay lazy."""
|
|
596
|
+
type_id = str(wire["typeId"])
|
|
597
|
+
fingerprint = str(wire["fingerprint"])
|
|
598
|
+
meta_raw = default_decode(blobs[int(wire["metaBlob"])])
|
|
599
|
+
if not isinstance(meta_raw, Mapping):
|
|
600
|
+
raise BoundaryError(f"value meta must decode to a mapping, got {type(meta_raw)}")
|
|
601
|
+
meta_entries = {
|
|
602
|
+
str(key): val for key, val in cast("Mapping[object, object]", meta_raw).items()
|
|
603
|
+
}
|
|
604
|
+
meta = ValueMeta(meta_entries)
|
|
605
|
+
if "elements" in wire:
|
|
606
|
+
# Recursive container descriptor: rebuild the child envelopes
|
|
607
|
+
# and their positional or homogeneous parent. Identity crosses
|
|
608
|
+
# verbatim like every fingerprint does (hazard H4).
|
|
609
|
+
positions = parse_tuple_type_id(type_id)
|
|
610
|
+
if parse_list_type_id(type_id) is None and positions is None:
|
|
611
|
+
raise BoundaryError(f"elements on a non-container type id: {type_id!r}")
|
|
612
|
+
child_values: list[Value] = []
|
|
613
|
+
total_size = 0
|
|
614
|
+
total_ms = 0.0
|
|
615
|
+
all_declared = True
|
|
616
|
+
elements = cast("Sequence[Mapping[str, Any]]", wire["elements"])
|
|
617
|
+
for child_wire in elements:
|
|
618
|
+
child, child_stat = self.decode(child_wire, blobs, consumed_segments)
|
|
619
|
+
child_values.append(child)
|
|
620
|
+
total_size += child_stat.size_bytes
|
|
621
|
+
total_ms += child_stat.codec_ms
|
|
622
|
+
all_declared = all_declared and child_stat.declared_codec
|
|
623
|
+
if positions is not None and positions != tuple(
|
|
624
|
+
child.type_id for child in child_values
|
|
625
|
+
):
|
|
626
|
+
raise BoundaryError("tuple child types do not match their envelope positions")
|
|
627
|
+
payload = (
|
|
628
|
+
ListPayload(tuple(child_values))
|
|
629
|
+
if positions is None
|
|
630
|
+
else TuplePayload(tuple(child_values))
|
|
631
|
+
)
|
|
632
|
+
container_value = Value(
|
|
633
|
+
type_id=type_id,
|
|
634
|
+
fingerprint=fingerprint,
|
|
635
|
+
meta=meta,
|
|
636
|
+
payload=payload,
|
|
637
|
+
)
|
|
638
|
+
return container_value, TransferStat(
|
|
639
|
+
payload.transport, total_size, total_ms, all_declared, False
|
|
640
|
+
)
|
|
641
|
+
payload_wire = cast("Mapping[str, Any]", wire["payload"])
|
|
642
|
+
transport = str(payload_wire["transport"])
|
|
643
|
+
started = time.perf_counter()
|
|
644
|
+
data = b""
|
|
645
|
+
segment: SharedMemory | None = None
|
|
646
|
+
segment_view: memoryview | None = None
|
|
647
|
+
size_bytes: int
|
|
648
|
+
if transport == "shm":
|
|
649
|
+
if not self._accept_shm:
|
|
650
|
+
raise BoundaryError(
|
|
651
|
+
"shm payloads are not accepted over this boundary; "
|
|
652
|
+
"the peers do not share memory"
|
|
653
|
+
)
|
|
654
|
+
name = str(payload_wire["segment"])
|
|
655
|
+
raw_size = payload_wire["size"]
|
|
656
|
+
if type(raw_size) is not int:
|
|
657
|
+
raise BoundaryError("shared-memory payload size must be an integer")
|
|
658
|
+
size_bytes = raw_size
|
|
659
|
+
segment, segment_view = _open_segment(name, size_bytes)
|
|
660
|
+
consumed_segments.append(name)
|
|
661
|
+
elif transport == "cas":
|
|
662
|
+
if not self._cas:
|
|
663
|
+
raise BoundaryError("cas payloads were not negotiated over this boundary")
|
|
664
|
+
digest = str(payload_wire["digest"])
|
|
665
|
+
raw_size = payload_wire["size"]
|
|
666
|
+
if type(raw_size) is not int or raw_size < 0:
|
|
667
|
+
raise BoundaryError("cas payload size must be a non-negative integer")
|
|
668
|
+
size_bytes = raw_size
|
|
669
|
+
if "blob" in payload_wire:
|
|
670
|
+
data = blobs[int(payload_wire["blob"])]
|
|
671
|
+
if len(data) != size_bytes or stable_hash([data]) != digest:
|
|
672
|
+
raise BoundaryError(f"cas payload does not match its digest ({digest})")
|
|
673
|
+
# Mirror the sender's retention decision verbatim (missing
|
|
674
|
+
# means retained, the pre-budget wire shape).
|
|
675
|
+
if bool(payload_wire.get("retained", True)):
|
|
676
|
+
self._retain(digest, data)
|
|
677
|
+
else:
|
|
678
|
+
cached = self._conversation.get(digest)
|
|
679
|
+
if cached is None:
|
|
680
|
+
raise BoundaryError(
|
|
681
|
+
"peer referenced a cas digest that never crossed "
|
|
682
|
+
f"this conversation: {digest}"
|
|
683
|
+
)
|
|
684
|
+
if len(cached) != size_bytes:
|
|
685
|
+
raise BoundaryError(
|
|
686
|
+
f"cas payload has size {len(cached)}, expected {size_bytes}"
|
|
687
|
+
)
|
|
688
|
+
data = cached
|
|
689
|
+
elif transport == "persistentCas":
|
|
690
|
+
if not self._persistent:
|
|
691
|
+
raise BoundaryError("persistentCas payloads were not negotiated over this boundary")
|
|
692
|
+
store = self._value_store
|
|
693
|
+
assert store is not None # enable_persistent_cas required one
|
|
694
|
+
digest = str(payload_wire["digest"])
|
|
695
|
+
raw_size = payload_wire["size"]
|
|
696
|
+
if type(raw_size) is not int or raw_size < 0:
|
|
697
|
+
raise BoundaryError("persistentCas payload size must be a non-negative integer")
|
|
698
|
+
size_bytes = raw_size
|
|
699
|
+
stored = store.get(digest)
|
|
700
|
+
if stored is None:
|
|
701
|
+
raise PersistentBlobMissing(
|
|
702
|
+
f"persistentCas blob {digest} is not in the local value store"
|
|
703
|
+
)
|
|
704
|
+
if len(stored) != size_bytes:
|
|
705
|
+
raise BoundaryError(
|
|
706
|
+
f"persistentCas blob {digest} has size {len(stored)}, expected {size_bytes}"
|
|
707
|
+
)
|
|
708
|
+
data = stored
|
|
709
|
+
elif transport == "inline":
|
|
710
|
+
data = blobs[int(payload_wire["blob"])]
|
|
711
|
+
size_bytes = len(data)
|
|
712
|
+
else:
|
|
713
|
+
raise BoundaryError(f"unknown payload transport: {transport!r}")
|
|
714
|
+
spec = self._registry.spec(type_id) if type_id in self._registry else None
|
|
715
|
+
if spec is not None and (
|
|
716
|
+
spec.validate_encoded is not None or spec.validate_encoded_buffer is not None
|
|
717
|
+
):
|
|
718
|
+
if segment_view is not None:
|
|
719
|
+
validation_view = segment_view.toreadonly()
|
|
720
|
+
if validation_view.nbytes != size_bytes:
|
|
721
|
+
sized_view = validation_view[:size_bytes]
|
|
722
|
+
validation_view.release()
|
|
723
|
+
validation_view = sized_view
|
|
724
|
+
validation_data: bytes | memoryview = validation_view
|
|
725
|
+
else:
|
|
726
|
+
validation_view = None
|
|
727
|
+
validation_data = data
|
|
728
|
+
try:
|
|
729
|
+
if spec.validate_encoded_buffer is not None:
|
|
730
|
+
if isinstance(validation_data, memoryview):
|
|
731
|
+
spec.validate_encoded_buffer(validation_data, meta.entries)
|
|
732
|
+
else:
|
|
733
|
+
with memoryview(validation_data) as inline_view:
|
|
734
|
+
spec.validate_encoded_buffer(inline_view, meta.entries)
|
|
735
|
+
else:
|
|
736
|
+
assert spec.validate_encoded is not None
|
|
737
|
+
spec.validate_encoded(bytes(validation_data), meta.entries)
|
|
738
|
+
except Exception as exc:
|
|
739
|
+
if validation_view is not None:
|
|
740
|
+
validation_view.release()
|
|
741
|
+
validation_view = None
|
|
742
|
+
if segment_view is not None:
|
|
743
|
+
segment_view.release()
|
|
744
|
+
segment_view = None
|
|
745
|
+
assert segment is not None
|
|
746
|
+
segment.close()
|
|
747
|
+
segment = None
|
|
748
|
+
raise EncodedPayloadValidationError(
|
|
749
|
+
f"encoded payload validation failed for type '{type_id}': {exc}"
|
|
750
|
+
) from exc
|
|
751
|
+
if validation_view is not None:
|
|
752
|
+
validation_view.release()
|
|
753
|
+
codec_ms = (time.perf_counter() - started) * 1000.0
|
|
754
|
+
decoder: Callable[[bytes], object] | None = None
|
|
755
|
+
declared = False
|
|
756
|
+
if spec is not None:
|
|
757
|
+
decoder = spec.decode
|
|
758
|
+
declared = spec.declared_codec
|
|
759
|
+
value = Value(
|
|
760
|
+
type_id=type_id,
|
|
761
|
+
fingerprint=fingerprint,
|
|
762
|
+
meta=ValueMeta(encoded_storage_meta(meta.entries, size_bytes)),
|
|
763
|
+
payload=(
|
|
764
|
+
EncodedPayload.from_buffer(
|
|
765
|
+
type_id,
|
|
766
|
+
segment_view,
|
|
767
|
+
decoder,
|
|
768
|
+
transport,
|
|
769
|
+
cast(SharedMemory, segment).close,
|
|
770
|
+
size_bytes,
|
|
771
|
+
)
|
|
772
|
+
if segment_view is not None
|
|
773
|
+
else EncodedPayload(type_id, data, decoder, transport)
|
|
774
|
+
),
|
|
775
|
+
)
|
|
776
|
+
return value, TransferStat(transport, size_bytes, codec_ms, declared, False)
|
|
777
|
+
|
|
778
|
+
|
|
779
|
+
def _stat_to_wire(stat: TransferStat) -> dict[str, object]:
|
|
780
|
+
wire: dict[str, object] = {
|
|
781
|
+
"transport": stat.transport,
|
|
782
|
+
"sizeBytes": stat.size_bytes,
|
|
783
|
+
"codecMs": stat.codec_ms,
|
|
784
|
+
"declaredCodec": stat.declared_codec,
|
|
785
|
+
"reused": stat.reused,
|
|
786
|
+
}
|
|
787
|
+
if stat.network_bytes:
|
|
788
|
+
wire["networkBytes"] = stat.network_bytes
|
|
789
|
+
wire["transferMs"] = stat.transfer_ms
|
|
790
|
+
return wire
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
def stat_from_wire(wire: Mapping[str, Any]) -> TransferStat:
|
|
794
|
+
return TransferStat(
|
|
795
|
+
transport=str(wire["transport"]),
|
|
796
|
+
size_bytes=int(wire["sizeBytes"]),
|
|
797
|
+
codec_ms=float(wire["codecMs"]),
|
|
798
|
+
declared_codec=bool(wire["declaredCodec"]),
|
|
799
|
+
reused=bool(wire["reused"]),
|
|
800
|
+
network_bytes=int(wire.get("networkBytes", 0)),
|
|
801
|
+
transfer_ms=float(wire.get("transferMs", 0.0)),
|
|
802
|
+
)
|
|
803
|
+
|
|
804
|
+
|
|
805
|
+
def encode_invocation(
|
|
806
|
+
codec: ValueCodec, invocation: Invocation
|
|
807
|
+
) -> tuple[dict[str, object], list[bytes], list[SharedMemory], dict[str, TransferStat]]:
|
|
808
|
+
"""Build an ``invoke`` frame. The effective schema crosses in the schema
|
|
809
|
+
wire format - the same format the hello handshake and the server API use
|
|
810
|
+
(hazard H1: one interface description everywhere)."""
|
|
811
|
+
blobs: list[bytes] = []
|
|
812
|
+
segments: list[SharedMemory] = []
|
|
813
|
+
stats: dict[str, TransferStat] = {}
|
|
814
|
+
inputs_wire: dict[str, object] = {}
|
|
815
|
+
header: dict[str, object] = {
|
|
816
|
+
"type": "invoke",
|
|
817
|
+
"invocationId": invocation.invocation_id,
|
|
818
|
+
"jobRef": invocation.job_ref or invocation.invocation_id,
|
|
819
|
+
"attemptId": invocation.attempt_id,
|
|
820
|
+
"nodeId": invocation.node_id,
|
|
821
|
+
"nodeType": invocation.node_type,
|
|
822
|
+
"effectiveSchema": schema_to_wire(invocation.effective_schema),
|
|
823
|
+
"outputMembers": [
|
|
824
|
+
[family_id, list(suffixes)] for family_id, suffixes in invocation.output_members
|
|
825
|
+
],
|
|
826
|
+
"inputs": inputs_wire,
|
|
827
|
+
}
|
|
828
|
+
if invocation.arm is not None:
|
|
829
|
+
header["arm"] = invocation.arm
|
|
830
|
+
if invocation.expected_execution_identity is not None:
|
|
831
|
+
header["expectedExecutionIdentity"] = invocation.expected_execution_identity
|
|
832
|
+
if invocation.extension_snapshot_digest is not None:
|
|
833
|
+
header["extensionSnapshotDigest"] = invocation.extension_snapshot_digest
|
|
834
|
+
if invocation.asset_folders is not None:
|
|
835
|
+
header["assetFolders"] = invocation.asset_folders.to_wire()
|
|
836
|
+
if invocation.connected_undemanded_inputs:
|
|
837
|
+
header["connectedUndemandedInputs"] = list(invocation.connected_undemanded_inputs)
|
|
838
|
+
if invocation.export_snapshot is not None:
|
|
839
|
+
snapshot_wire: dict[str, object] = {}
|
|
840
|
+
if invocation.export_snapshot.prompt is not None:
|
|
841
|
+
snapshot_wire["prompt"] = dict(invocation.export_snapshot.prompt)
|
|
842
|
+
if invocation.export_snapshot.extra_pnginfo is not None:
|
|
843
|
+
snapshot_wire["extraPnginfo"] = dict(invocation.export_snapshot.extra_pnginfo)
|
|
844
|
+
header["exportSnapshot"] = snapshot_wire
|
|
845
|
+
header["fp8Matmul"] = invocation.fp8_matmul
|
|
846
|
+
if invocation.diffusion_dtype is not None:
|
|
847
|
+
header["componentDtypes"] = {
|
|
848
|
+
"diffusion": invocation.diffusion_dtype,
|
|
849
|
+
"textEncoder": invocation.text_dtype,
|
|
850
|
+
"vae": invocation.vae_dtype,
|
|
851
|
+
}
|
|
852
|
+
header["attentionPolicy"] = invocation.attention_policy
|
|
853
|
+
if invocation.saved_file_previews_enabled:
|
|
854
|
+
header["savedFilePreviewsEnabled"] = True
|
|
855
|
+
if invocation.preview_mode != "off":
|
|
856
|
+
header["previewMode"] = invocation.preview_mode
|
|
857
|
+
if invocation.preview_animation != "ring":
|
|
858
|
+
header["previewAnimation"] = invocation.preview_animation
|
|
859
|
+
if invocation.attention_route_token is not None:
|
|
860
|
+
from dinkster_protocol import attention_route_token_to_wire
|
|
861
|
+
|
|
862
|
+
header["attentionRouteToken"] = attention_route_token_to_wire(
|
|
863
|
+
invocation.attention_route_token
|
|
864
|
+
)
|
|
865
|
+
if invocation.single_job_multi_gpu is not None:
|
|
866
|
+
raise BoundaryError("host single-job multi-GPU selection was not resolved by its pool")
|
|
867
|
+
if invocation.single_job_multi_gpu_execution is not None:
|
|
868
|
+
from dinkster_protocol import single_job_multi_gpu_execution_to_wire
|
|
869
|
+
|
|
870
|
+
header["singleJobMultiGpuExecution"] = single_job_multi_gpu_execution_to_wire(
|
|
871
|
+
invocation.single_job_multi_gpu_execution
|
|
872
|
+
)
|
|
873
|
+
if invocation.media_sources:
|
|
874
|
+
header["mediaSources"] = [
|
|
875
|
+
{
|
|
876
|
+
"digest": source.digest,
|
|
877
|
+
"kind": source.kind,
|
|
878
|
+
"mediaType": source.media_type,
|
|
879
|
+
"extension": source.extension,
|
|
880
|
+
"byteSize": source.byte_size,
|
|
881
|
+
}
|
|
882
|
+
for source in invocation.media_sources
|
|
883
|
+
]
|
|
884
|
+
conversation_checkpoint = codec.conversation_checkpoint()
|
|
885
|
+
try:
|
|
886
|
+
for input_id, value in invocation.inputs.items():
|
|
887
|
+
wire, stat = codec.encode(value, blobs, segments)
|
|
888
|
+
inputs_wire[input_id] = wire
|
|
889
|
+
stats[input_id] = stat
|
|
890
|
+
except Exception:
|
|
891
|
+
for segment in segments:
|
|
892
|
+
release_segment(segment)
|
|
893
|
+
codec.restore_conversation(conversation_checkpoint)
|
|
894
|
+
codec.take_pending_store_blobs()
|
|
895
|
+
raise
|
|
896
|
+
return header, blobs, segments, stats
|
|
897
|
+
|
|
898
|
+
|
|
899
|
+
def decode_invocation(
|
|
900
|
+
codec: ValueCodec, header: Mapping[str, Any], blobs: Sequence[bytes], consumed: list[str]
|
|
901
|
+
) -> Invocation:
|
|
902
|
+
from dinkster_protocol import (
|
|
903
|
+
attention_route_token_from_wire,
|
|
904
|
+
resolve_attention_runtime_status,
|
|
905
|
+
single_job_multi_gpu_execution_from_wire,
|
|
906
|
+
validate_attention_policy,
|
|
907
|
+
)
|
|
908
|
+
|
|
909
|
+
job_ref_raw = header.get("jobRef")
|
|
910
|
+
if type(job_ref_raw) is not str or not job_ref_raw:
|
|
911
|
+
raise BoundaryError("invocation jobRef must be a non-empty string")
|
|
912
|
+
attempt_id_raw = header.get("attemptId")
|
|
913
|
+
if type(attempt_id_raw) is not int or attempt_id_raw < 1:
|
|
914
|
+
raise BoundaryError("invocation attemptId must be a positive integer")
|
|
915
|
+
inputs: dict[str, Value] = {}
|
|
916
|
+
for input_id, wire in cast("Mapping[str, Any]", header["inputs"]).items():
|
|
917
|
+
try:
|
|
918
|
+
value, _ = codec.decode(cast("Mapping[str, Any]", wire), blobs, consumed)
|
|
919
|
+
except PersistentBlobMissing as exc:
|
|
920
|
+
raise PersistentBlobMissing(
|
|
921
|
+
f"input '{input_id}' of node '{header.get('nodeId')}': {exc}"
|
|
922
|
+
) from exc
|
|
923
|
+
inputs[str(input_id)] = value
|
|
924
|
+
output_members = tuple(
|
|
925
|
+
(str(family_id), tuple(str(suffix) for suffix in suffixes))
|
|
926
|
+
for family_id, suffixes in header.get("outputMembers", ())
|
|
927
|
+
)
|
|
928
|
+
arm_raw = header.get("arm")
|
|
929
|
+
if "arm" in header and (not isinstance(arm_raw, str) or not arm_raw):
|
|
930
|
+
raise BoundaryError("invocation arm must be a non-empty string when present")
|
|
931
|
+
identity_raw = header.get("expectedExecutionIdentity")
|
|
932
|
+
if "expectedExecutionIdentity" in header and (
|
|
933
|
+
not isinstance(identity_raw, str) or not identity_raw
|
|
934
|
+
):
|
|
935
|
+
raise BoundaryError(
|
|
936
|
+
"invocation expectedExecutionIdentity must be a non-empty string when present"
|
|
937
|
+
)
|
|
938
|
+
extension_digest_raw = header.get("extensionSnapshotDigest")
|
|
939
|
+
if "extensionSnapshotDigest" in header and (
|
|
940
|
+
not is_extension_snapshot_digest(extension_digest_raw)
|
|
941
|
+
):
|
|
942
|
+
raise BoundaryError(
|
|
943
|
+
"invocation extensionSnapshotDigest must be a sha256 digest when present"
|
|
944
|
+
)
|
|
945
|
+
export_snapshot_raw = header.get("exportSnapshot")
|
|
946
|
+
try:
|
|
947
|
+
asset_folders = (
|
|
948
|
+
AssetFolderContext.from_wire(header["assetFolders"])
|
|
949
|
+
if "assetFolders" in header
|
|
950
|
+
else None
|
|
951
|
+
)
|
|
952
|
+
except ValueError as exc:
|
|
953
|
+
raise BoundaryError(f"invalid invocation assetFolders: {exc}") from exc
|
|
954
|
+
export_snapshot = None
|
|
955
|
+
if "exportSnapshot" in header:
|
|
956
|
+
if not isinstance(export_snapshot_raw, Mapping):
|
|
957
|
+
raise BoundaryError("invocation exportSnapshot must be an object")
|
|
958
|
+
export_snapshot_wire = cast("Mapping[str, Any]", export_snapshot_raw)
|
|
959
|
+
if set(export_snapshot_wire) - {"prompt", "extraPnginfo"}:
|
|
960
|
+
raise BoundaryError("invocation exportSnapshot has unknown fields")
|
|
961
|
+
for field_name in ("prompt", "extraPnginfo"):
|
|
962
|
+
if field_name in export_snapshot_wire and not isinstance(
|
|
963
|
+
export_snapshot_wire[field_name], Mapping
|
|
964
|
+
):
|
|
965
|
+
raise BoundaryError(f"invocation exportSnapshot {field_name} must be an object")
|
|
966
|
+
try:
|
|
967
|
+
export_snapshot = ExportSnapshot(
|
|
968
|
+
prompt=cast("Mapping[str, object] | None", export_snapshot_wire.get("prompt")),
|
|
969
|
+
extra_pnginfo=cast(
|
|
970
|
+
"Mapping[str, object] | None", export_snapshot_wire.get("extraPnginfo")
|
|
971
|
+
),
|
|
972
|
+
)
|
|
973
|
+
except ValueError as exc:
|
|
974
|
+
raise BoundaryError(f"invalid invocation exportSnapshot: {exc}") from exc
|
|
975
|
+
fp8_matmul_raw = header.get("fp8Matmul", False)
|
|
976
|
+
if not isinstance(fp8_matmul_raw, bool):
|
|
977
|
+
raise BoundaryError("invocation fp8Matmul must be a boolean")
|
|
978
|
+
if fp8_matmul_raw and identity_raw is None:
|
|
979
|
+
raise BoundaryError("invocation fp8Matmul requires expectedExecutionIdentity")
|
|
980
|
+
component_dtypes_raw = header.get("componentDtypes")
|
|
981
|
+
component_dtypes: tuple[str | None, str | None, str | None] = (None, None, None)
|
|
982
|
+
if component_dtypes_raw is not None:
|
|
983
|
+
if not isinstance(component_dtypes_raw, Mapping) or set(component_dtypes_raw) != {
|
|
984
|
+
"diffusion",
|
|
985
|
+
"textEncoder",
|
|
986
|
+
"vae",
|
|
987
|
+
}:
|
|
988
|
+
raise BoundaryError("invocation componentDtypes must contain all components")
|
|
989
|
+
values = cast("Mapping[str, object]", component_dtypes_raw)
|
|
990
|
+
component_dtypes = (
|
|
991
|
+
cast("str", values["diffusion"]),
|
|
992
|
+
cast("str", values["textEncoder"]),
|
|
993
|
+
cast("str", values["vae"]),
|
|
994
|
+
)
|
|
995
|
+
if identity_raw is None or not all(
|
|
996
|
+
isinstance(dtype, str) and dtype for dtype in component_dtypes
|
|
997
|
+
):
|
|
998
|
+
raise BoundaryError(
|
|
999
|
+
"invocation componentDtypes must be non-empty strings and require identity"
|
|
1000
|
+
)
|
|
1001
|
+
try:
|
|
1002
|
+
attention_policy = validate_attention_policy(header.get("attentionPolicy", "auto"))
|
|
1003
|
+
attention_token = (
|
|
1004
|
+
attention_route_token_from_wire(header["attentionRouteToken"])
|
|
1005
|
+
if "attentionRouteToken" in header
|
|
1006
|
+
else None
|
|
1007
|
+
)
|
|
1008
|
+
resolve_attention_runtime_status(attention_policy, attention_token)
|
|
1009
|
+
except (TypeError, ValueError) as exc:
|
|
1010
|
+
raise BoundaryError(f"malformed invocation attention routing: {exc}") from exc
|
|
1011
|
+
try:
|
|
1012
|
+
single_job_multi_gpu_execution = (
|
|
1013
|
+
single_job_multi_gpu_execution_from_wire(header["singleJobMultiGpuExecution"])
|
|
1014
|
+
if "singleJobMultiGpuExecution" in header
|
|
1015
|
+
else None
|
|
1016
|
+
)
|
|
1017
|
+
except (TypeError, ValueError) as exc:
|
|
1018
|
+
raise BoundaryError(f"malformed invocation single-job multi-GPU execution: {exc}") from exc
|
|
1019
|
+
try:
|
|
1020
|
+
preview_mode = validate_preview_mode(header.get("previewMode", "off"))
|
|
1021
|
+
except ValueError as exc:
|
|
1022
|
+
raise BoundaryError(f"malformed invocation preview mode: {exc}") from exc
|
|
1023
|
+
try:
|
|
1024
|
+
preview_animation = validate_preview_animation(header.get("previewAnimation", "ring"))
|
|
1025
|
+
except ValueError as exc:
|
|
1026
|
+
raise BoundaryError(f"malformed invocation preview animation: {exc}") from exc
|
|
1027
|
+
undemanded_raw = cast("object", header.get("connectedUndemandedInputs", ()))
|
|
1028
|
+
saved_file_previews_enabled = header.get("savedFilePreviewsEnabled", False)
|
|
1029
|
+
if type(saved_file_previews_enabled) is not bool:
|
|
1030
|
+
raise BoundaryError("savedFilePreviewsEnabled must be boolean")
|
|
1031
|
+
if (
|
|
1032
|
+
not isinstance(undemanded_raw, (list, tuple))
|
|
1033
|
+
or not all(isinstance(item, str) for item in cast("Sequence[object]", undemanded_raw))
|
|
1034
|
+
or len(set(cast("Sequence[object]", undemanded_raw)))
|
|
1035
|
+
!= len(cast("Sequence[object]", undemanded_raw))
|
|
1036
|
+
):
|
|
1037
|
+
raise BoundaryError("invocation connectedUndemandedInputs must be unique strings")
|
|
1038
|
+
media_sources_raw = cast("object", header.get("mediaSources", ()))
|
|
1039
|
+
if not isinstance(media_sources_raw, (list, tuple)):
|
|
1040
|
+
raise BoundaryError("invocation mediaSources must be a list")
|
|
1041
|
+
media_sources: list[MediaSourceAuthority] = []
|
|
1042
|
+
for raw_object in cast("Sequence[object]", media_sources_raw):
|
|
1043
|
+
if not isinstance(raw_object, Mapping) or set(
|
|
1044
|
+
cast("Mapping[object, object]", raw_object)
|
|
1045
|
+
) != {
|
|
1046
|
+
"digest",
|
|
1047
|
+
"kind",
|
|
1048
|
+
"mediaType",
|
|
1049
|
+
"extension",
|
|
1050
|
+
"byteSize",
|
|
1051
|
+
}:
|
|
1052
|
+
raise BoundaryError("invocation mediaSources entries have invalid fields")
|
|
1053
|
+
raw = cast("Mapping[str, object]", raw_object)
|
|
1054
|
+
try:
|
|
1055
|
+
media_sources.append(
|
|
1056
|
+
MediaSourceAuthority(
|
|
1057
|
+
digest=cast("str", raw["digest"]),
|
|
1058
|
+
kind=cast("str", raw["kind"]),
|
|
1059
|
+
media_type=cast("str", raw["mediaType"]),
|
|
1060
|
+
extension=cast("str", raw["extension"]),
|
|
1061
|
+
byte_size=cast("int", raw["byteSize"]),
|
|
1062
|
+
)
|
|
1063
|
+
)
|
|
1064
|
+
except (TypeError, ValueError) as exc:
|
|
1065
|
+
raise BoundaryError(f"invalid invocation mediaSources entry: {exc}") from exc
|
|
1066
|
+
try:
|
|
1067
|
+
return Invocation(
|
|
1068
|
+
invocation_id=str(header["invocationId"]),
|
|
1069
|
+
node_id=str(header["nodeId"]),
|
|
1070
|
+
node_type=str(header["nodeType"]),
|
|
1071
|
+
inputs=inputs,
|
|
1072
|
+
effective_schema=schema_from_wire(
|
|
1073
|
+
dict(cast("Mapping[str, Any]", header["effectiveSchema"]))
|
|
1074
|
+
),
|
|
1075
|
+
job_ref=job_ref_raw,
|
|
1076
|
+
attempt_id=attempt_id_raw,
|
|
1077
|
+
output_members=output_members,
|
|
1078
|
+
connected_undemanded_inputs=tuple(cast("Sequence[str]", undemanded_raw)),
|
|
1079
|
+
arm=arm_raw,
|
|
1080
|
+
expected_execution_identity=identity_raw,
|
|
1081
|
+
extension_snapshot_digest=extension_digest_raw,
|
|
1082
|
+
export_snapshot=export_snapshot,
|
|
1083
|
+
asset_folders=asset_folders,
|
|
1084
|
+
fp8_matmul=fp8_matmul_raw,
|
|
1085
|
+
diffusion_dtype=component_dtypes[0],
|
|
1086
|
+
text_dtype=component_dtypes[1],
|
|
1087
|
+
vae_dtype=component_dtypes[2],
|
|
1088
|
+
attention_policy=attention_policy,
|
|
1089
|
+
single_job_multi_gpu_execution=single_job_multi_gpu_execution,
|
|
1090
|
+
attention_route_token=attention_token,
|
|
1091
|
+
media_sources=tuple(media_sources),
|
|
1092
|
+
saved_file_previews_enabled=saved_file_previews_enabled,
|
|
1093
|
+
preview_mode=preview_mode,
|
|
1094
|
+
preview_animation=preview_animation,
|
|
1095
|
+
)
|
|
1096
|
+
except ValueError as exc:
|
|
1097
|
+
raise BoundaryError(f"invalid invocation: {exc}") from exc
|
|
1098
|
+
|
|
1099
|
+
|
|
1100
|
+
def encode_result(
|
|
1101
|
+
codec: ValueCodec,
|
|
1102
|
+
result: InvocationResult,
|
|
1103
|
+
invocation_id: str,
|
|
1104
|
+
execute_ms: float,
|
|
1105
|
+
) -> tuple[dict[str, object], list[bytes], list[SharedMemory]]:
|
|
1106
|
+
from .saved_artifacts import MAX_SAVED_ARTIFACTS, validate_candidate
|
|
1107
|
+
|
|
1108
|
+
if len(result.artifact_candidates) > MAX_SAVED_ARTIFACTS:
|
|
1109
|
+
raise BoundaryError(f"result artifactCandidates exceeds {MAX_SAVED_ARTIFACTS} entries")
|
|
1110
|
+
try:
|
|
1111
|
+
for candidate in result.artifact_candidates:
|
|
1112
|
+
validate_candidate(candidate)
|
|
1113
|
+
except AssetError as exc:
|
|
1114
|
+
raise BoundaryError(f"invalid result artifact candidate: {exc}") from exc
|
|
1115
|
+
header: dict[str, object] = {
|
|
1116
|
+
"type": "result",
|
|
1117
|
+
"invocationId": invocation_id,
|
|
1118
|
+
"executeMs": execute_ms,
|
|
1119
|
+
}
|
|
1120
|
+
if result.artifact_candidates:
|
|
1121
|
+
header["artifactCandidates"] = [
|
|
1122
|
+
{
|
|
1123
|
+
"nodeId": candidate.node_id,
|
|
1124
|
+
"filename": candidate.filename,
|
|
1125
|
+
"subfolder": candidate.subfolder,
|
|
1126
|
+
"type": candidate.folder_type,
|
|
1127
|
+
}
|
|
1128
|
+
for candidate in result.artifact_candidates
|
|
1129
|
+
]
|
|
1130
|
+
if result.artifacts:
|
|
1131
|
+
raise BoundaryError("workers cannot send authoritative saved artifact descriptors")
|
|
1132
|
+
if result.error is None and result.saved_artifact_claims:
|
|
1133
|
+
header["savedArtifactClaims"] = [dict(claim) for claim in result.saved_artifact_claims]
|
|
1134
|
+
decode_saved_artifact_claims(header)
|
|
1135
|
+
if result.error is None and result.saved_file_previews:
|
|
1136
|
+
header["savedFilePreviews"] = [
|
|
1137
|
+
{key: dict(cast("Mapping[str, object]", value)) for key, value in claim.items()}
|
|
1138
|
+
for claim in result.saved_file_previews
|
|
1139
|
+
]
|
|
1140
|
+
if result.expansion is not None:
|
|
1141
|
+
from dinkster_protocol import InvocationReturn, ReturnBatch
|
|
1142
|
+
|
|
1143
|
+
algebra_header, blobs, segments = encode_invocation_outcome(
|
|
1144
|
+
codec,
|
|
1145
|
+
InvocationReturn(ReturnBatch("scalar", (result.expansion,))),
|
|
1146
|
+
invocation_id,
|
|
1147
|
+
execute_ms,
|
|
1148
|
+
)
|
|
1149
|
+
header.update(algebra_header)
|
|
1150
|
+
_encode_preview_artifacts(header, blobs, result.preview_artifacts)
|
|
1151
|
+
return header, blobs, segments
|
|
1152
|
+
blobs: list[bytes] = []
|
|
1153
|
+
segments: list[SharedMemory] = []
|
|
1154
|
+
if result.error is not None:
|
|
1155
|
+
header["error"] = {
|
|
1156
|
+
"nodeId": result.error.node_id,
|
|
1157
|
+
"nodeType": result.error.node_type,
|
|
1158
|
+
"message": result.error.message,
|
|
1159
|
+
"traceback": result.error.traceback,
|
|
1160
|
+
"hints": [
|
|
1161
|
+
{
|
|
1162
|
+
"code": hint.code,
|
|
1163
|
+
"message": hint.message,
|
|
1164
|
+
**({"suggestion": hint.suggestion} if hint.suggestion is not None else {}),
|
|
1165
|
+
}
|
|
1166
|
+
for hint in result.error.hints
|
|
1167
|
+
],
|
|
1168
|
+
}
|
|
1169
|
+
return header, blobs, segments
|
|
1170
|
+
outputs_wire: dict[str, object] = {}
|
|
1171
|
+
stats_wire: dict[str, object] = {}
|
|
1172
|
+
conversation_checkpoint = codec.conversation_checkpoint()
|
|
1173
|
+
try:
|
|
1174
|
+
for output_id, value in (result.outputs or {}).items():
|
|
1175
|
+
wire, stat = codec.encode(value, blobs, segments)
|
|
1176
|
+
outputs_wire[output_id] = wire
|
|
1177
|
+
stats_wire[output_id] = _stat_to_wire(stat)
|
|
1178
|
+
except Exception:
|
|
1179
|
+
for segment in segments:
|
|
1180
|
+
release_segment(segment)
|
|
1181
|
+
codec.restore_conversation(conversation_checkpoint)
|
|
1182
|
+
codec.take_pending_store_blobs()
|
|
1183
|
+
raise
|
|
1184
|
+
header["outputs"] = outputs_wire
|
|
1185
|
+
header["outputStats"] = stats_wire
|
|
1186
|
+
_encode_preview_artifacts(header, blobs, result.preview_artifacts)
|
|
1187
|
+
return header, blobs, segments
|
|
1188
|
+
|
|
1189
|
+
|
|
1190
|
+
def _encode_preview_artifacts(
|
|
1191
|
+
header: dict[str, object], blobs: list[bytes], artifacts: Sequence[EncodedPreviewArtifact]
|
|
1192
|
+
) -> None:
|
|
1193
|
+
if not artifacts:
|
|
1194
|
+
return
|
|
1195
|
+
claims: list[dict[str, object]] = []
|
|
1196
|
+
for artifact in artifacts:
|
|
1197
|
+
claims.append({"name": artifact.name, "mediaType": artifact.media_type, "blob": len(blobs)})
|
|
1198
|
+
blobs.append(artifact.data)
|
|
1199
|
+
header["previewArtifacts"] = claims
|
|
1200
|
+
|
|
1201
|
+
|
|
1202
|
+
def _decode_preview_artifacts(
|
|
1203
|
+
header: Mapping[str, Any], blobs: Sequence[bytes]
|
|
1204
|
+
) -> tuple[EncodedPreviewArtifact, ...]:
|
|
1205
|
+
from dinkster_schema.reporting import capture_preview_artifacts, publish_preview_artifact
|
|
1206
|
+
|
|
1207
|
+
claims = header.get("previewArtifacts", [])
|
|
1208
|
+
if type(claims) is not list or len(claims) > 64:
|
|
1209
|
+
raise BoundaryError("invalid preview artifact batch")
|
|
1210
|
+
try:
|
|
1211
|
+
with capture_preview_artifacts() as artifacts:
|
|
1212
|
+
for position, claim in enumerate(claims):
|
|
1213
|
+
if type(claim) is not dict or set(claim) != {"name", "mediaType", "blob"}:
|
|
1214
|
+
raise BoundaryError("invalid preview artifact fields")
|
|
1215
|
+
index = claim["blob"]
|
|
1216
|
+
if (
|
|
1217
|
+
type(index) is not int
|
|
1218
|
+
or not 0 <= index < len(blobs)
|
|
1219
|
+
or index != len(blobs) - len(claims) + position
|
|
1220
|
+
):
|
|
1221
|
+
raise BoundaryError("invalid preview artifact blob index")
|
|
1222
|
+
publish_preview_artifact(
|
|
1223
|
+
blobs[index], name=claim["name"], media_type=claim["mediaType"]
|
|
1224
|
+
)
|
|
1225
|
+
return tuple(artifacts)
|
|
1226
|
+
except ValueError as exc:
|
|
1227
|
+
raise BoundaryError(str(exc)) from exc
|
|
1228
|
+
|
|
1229
|
+
|
|
1230
|
+
def decode_saved_artifact_claims(header: Mapping[str, Any]) -> tuple[Mapping[str, object], ...]:
|
|
1231
|
+
from dinkster_schema.reporting import attach_saved_artifact, capture_saved_artifacts
|
|
1232
|
+
from dinkster_values import SAVED_ARTIFACT_CLAIM_LIMIT
|
|
1233
|
+
|
|
1234
|
+
raw = header.get("savedArtifactClaims", [])
|
|
1235
|
+
if type(raw) is not list or len(raw) > SAVED_ARTIFACT_CLAIM_LIMIT:
|
|
1236
|
+
raise BoundaryError("savedArtifactClaims must be a bounded list")
|
|
1237
|
+
try:
|
|
1238
|
+
with capture_saved_artifacts() as claims:
|
|
1239
|
+
for claim in raw:
|
|
1240
|
+
if not isinstance(claim, Mapping):
|
|
1241
|
+
raise BoundaryError("saved artifact claim must be a mapping")
|
|
1242
|
+
attach_saved_artifact(claim)
|
|
1243
|
+
return tuple(claims)
|
|
1244
|
+
except (TypeError, ValueError) as exc:
|
|
1245
|
+
raise BoundaryError(str(exc)) from exc
|
|
1246
|
+
|
|
1247
|
+
|
|
1248
|
+
def decode_saved_file_previews(header: Mapping[str, Any]) -> tuple[Mapping[str, object], ...]:
|
|
1249
|
+
"""Optional malformed claims are omitted, never promoted to saved authority."""
|
|
1250
|
+
from dinkster_assets import AssetRef
|
|
1251
|
+
from dinkster_values import SAVED_ARTIFACT_CLAIM_LIMIT
|
|
1252
|
+
|
|
1253
|
+
from .preview_artifacts import SAVED_FILE_PREVIEW_LIMIT_BYTES
|
|
1254
|
+
|
|
1255
|
+
raw = header.get("savedFilePreviews", [])
|
|
1256
|
+
if type(raw) is not list or len(raw) > SAVED_ARTIFACT_CLAIM_LIMIT:
|
|
1257
|
+
return ()
|
|
1258
|
+
claims: list[Mapping[str, object]] = []
|
|
1259
|
+
for claim in raw:
|
|
1260
|
+
try:
|
|
1261
|
+
if not isinstance(claim, Mapping) or set(claim) != {"saved", "preview"}:
|
|
1262
|
+
continue
|
|
1263
|
+
saved = decode_saved_artifact_claims({"savedArtifactClaims": [claim["saved"]]})[0]
|
|
1264
|
+
preview = AssetRef.from_wire(claim["preview"])
|
|
1265
|
+
if (
|
|
1266
|
+
preview.size <= 0
|
|
1267
|
+
or preview.size > SAVED_FILE_PREVIEW_LIMIT_BYTES
|
|
1268
|
+
or preview.virtual_path != f"previews/{preview.digest[7:]}/{preview.name}"
|
|
1269
|
+
or "/" in preview.name
|
|
1270
|
+
or "\\" in preview.name
|
|
1271
|
+
or preview.name in (".", "..")
|
|
1272
|
+
):
|
|
1273
|
+
continue
|
|
1274
|
+
claims.append({"saved": saved, "preview": preview.to_wire()})
|
|
1275
|
+
except (AssetError, BoundaryError, TypeError, ValueError, KeyError):
|
|
1276
|
+
continue
|
|
1277
|
+
return tuple(claims)
|
|
1278
|
+
|
|
1279
|
+
|
|
1280
|
+
def decode_result_artifact_candidates(
|
|
1281
|
+
header: Mapping[str, Any],
|
|
1282
|
+
) -> tuple[SavedArtifactCandidate, ...]:
|
|
1283
|
+
"""Bound raw worker reports before host filesystem validation."""
|
|
1284
|
+
if "artifacts" in header:
|
|
1285
|
+
raise BoundaryError("worker sent a forged authoritative artifact descriptor")
|
|
1286
|
+
raw_candidates = header.get("artifactCandidates", [])
|
|
1287
|
+
if type(raw_candidates) is not list:
|
|
1288
|
+
raise BoundaryError("result artifactCandidates must be a list")
|
|
1289
|
+
from .saved_artifacts import (
|
|
1290
|
+
MAX_SAVED_ARTIFACTS,
|
|
1291
|
+
validate_candidate,
|
|
1292
|
+
)
|
|
1293
|
+
|
|
1294
|
+
if len(raw_candidates) > MAX_SAVED_ARTIFACTS:
|
|
1295
|
+
raise BoundaryError(f"result artifactCandidates exceeds {MAX_SAVED_ARTIFACTS} entries")
|
|
1296
|
+
candidates: list[SavedArtifactCandidate] = []
|
|
1297
|
+
for item in cast("list[object]", raw_candidates):
|
|
1298
|
+
if not isinstance(item, Mapping) or set(item) != {
|
|
1299
|
+
"nodeId",
|
|
1300
|
+
"filename",
|
|
1301
|
+
"subfolder",
|
|
1302
|
+
"type",
|
|
1303
|
+
}:
|
|
1304
|
+
raise BoundaryError("result artifact candidate has invalid fields")
|
|
1305
|
+
wire = cast("Mapping[str, object]", item)
|
|
1306
|
+
try:
|
|
1307
|
+
candidate = SavedArtifactCandidate(
|
|
1308
|
+
node_id=cast("str", wire["nodeId"]),
|
|
1309
|
+
filename=cast("str", wire["filename"]),
|
|
1310
|
+
subfolder=cast("str", wire["subfolder"]),
|
|
1311
|
+
folder_type=cast("str", wire["type"]),
|
|
1312
|
+
)
|
|
1313
|
+
validate_candidate(candidate)
|
|
1314
|
+
candidates.append(candidate)
|
|
1315
|
+
except (AssetError, TypeError, ValueError) as exc:
|
|
1316
|
+
raise BoundaryError(f"invalid result artifact candidate: {exc}") from exc
|
|
1317
|
+
return tuple(candidates)
|
|
1318
|
+
|
|
1319
|
+
|
|
1320
|
+
def encode_invocation_outcome(
|
|
1321
|
+
codec: ValueCodec,
|
|
1322
|
+
outcome: InvocationOutcome,
|
|
1323
|
+
invocation_id: str,
|
|
1324
|
+
execute_ms: float,
|
|
1325
|
+
) -> tuple[dict[str, object], list[bytes], list[SharedMemory]]:
|
|
1326
|
+
"""Encode invocation return units and their typed present values."""
|
|
1327
|
+
from dinkster_protocol import (
|
|
1328
|
+
RESULT_ALGEBRA_MAX_COUNT,
|
|
1329
|
+
RESULT_ALGEBRA_MAX_DOCUMENT_BYTES,
|
|
1330
|
+
)
|
|
1331
|
+
|
|
1332
|
+
if type(invocation_id) is not str:
|
|
1333
|
+
raise BoundaryError("typed result invocation id must be a string")
|
|
1334
|
+
_finite_number(execute_ms, "typed result executeMs")
|
|
1335
|
+
values: list[object] = []
|
|
1336
|
+
stats: list[object] = []
|
|
1337
|
+
blobs: list[bytes] = []
|
|
1338
|
+
segments: list[SharedMemory] = []
|
|
1339
|
+
value_budget = [RESULT_ALGEBRA_MAX_COUNT]
|
|
1340
|
+
conversation_checkpoint = codec.conversation_checkpoint()
|
|
1341
|
+
try:
|
|
1342
|
+
with codec.suspend_persistent_cas():
|
|
1343
|
+
document = _outcome_to_document(
|
|
1344
|
+
outcome, codec, values, stats, blobs, segments, value_budget
|
|
1345
|
+
)
|
|
1346
|
+
_validate_json_shape(document)
|
|
1347
|
+
document_json = json.dumps(
|
|
1348
|
+
document,
|
|
1349
|
+
sort_keys=True,
|
|
1350
|
+
separators=(",", ":"),
|
|
1351
|
+
ensure_ascii=True,
|
|
1352
|
+
allow_nan=False,
|
|
1353
|
+
)
|
|
1354
|
+
if len(document_json.encode("ascii")) > RESULT_ALGEBRA_MAX_DOCUMENT_BYTES:
|
|
1355
|
+
raise BoundaryError("result algebra document is excessive")
|
|
1356
|
+
header: dict[str, object] = {
|
|
1357
|
+
"type": "result",
|
|
1358
|
+
"invocationId": invocation_id,
|
|
1359
|
+
"executeMs": execute_ms,
|
|
1360
|
+
"resultAlgebra": {
|
|
1361
|
+
"document": document_json,
|
|
1362
|
+
"values": values,
|
|
1363
|
+
"valueStats": stats,
|
|
1364
|
+
},
|
|
1365
|
+
}
|
|
1366
|
+
framed = dict(header)
|
|
1367
|
+
framed["blobs"] = [len(blob) for blob in blobs]
|
|
1368
|
+
if len(json.dumps(framed, separators=(",", ":")).encode("utf-8")) > _MAX_HEADER_BYTES:
|
|
1369
|
+
raise BoundaryError("result algebra frame header is excessive")
|
|
1370
|
+
except Exception as exc:
|
|
1371
|
+
for segment in segments:
|
|
1372
|
+
release_segment(segment)
|
|
1373
|
+
codec.restore_conversation(conversation_checkpoint)
|
|
1374
|
+
raise BoundaryError("result algebra document cannot be encoded") from exc
|
|
1375
|
+
return header, blobs, segments
|
|
1376
|
+
|
|
1377
|
+
|
|
1378
|
+
def _outcome_to_document(
|
|
1379
|
+
outcome: object,
|
|
1380
|
+
codec: ValueCodec,
|
|
1381
|
+
values: list[object],
|
|
1382
|
+
stats: list[object],
|
|
1383
|
+
blobs: list[bytes],
|
|
1384
|
+
segments: list[SharedMemory],
|
|
1385
|
+
value_budget: list[int],
|
|
1386
|
+
) -> dict[str, object]:
|
|
1387
|
+
from dinkster_protocol import InvocationFailure, InvocationReturn
|
|
1388
|
+
from dinkster_protocol.result_document import return_unit_to_document
|
|
1389
|
+
|
|
1390
|
+
if isinstance(outcome, InvocationFailure):
|
|
1391
|
+
error = outcome.error
|
|
1392
|
+
return {
|
|
1393
|
+
"kind": "error",
|
|
1394
|
+
"error": {
|
|
1395
|
+
"hints": [
|
|
1396
|
+
{"code": h.code, "message": h.message, "suggestion": h.suggestion}
|
|
1397
|
+
for h in error.hints
|
|
1398
|
+
],
|
|
1399
|
+
"message": error.message,
|
|
1400
|
+
"nodeId": error.node_id,
|
|
1401
|
+
"nodeType": error.node_type,
|
|
1402
|
+
"traceback": error.traceback,
|
|
1403
|
+
},
|
|
1404
|
+
}
|
|
1405
|
+
if not isinstance(outcome, InvocationReturn):
|
|
1406
|
+
raise BoundaryError("invalid invocation outcome")
|
|
1407
|
+
|
|
1408
|
+
def encode_present(value: Value) -> int:
|
|
1409
|
+
_preflight_value_tree(value, value_budget)
|
|
1410
|
+
wire, stat = codec.encode(value, blobs, segments)
|
|
1411
|
+
index = len(values)
|
|
1412
|
+
values.append(wire)
|
|
1413
|
+
stats.append(_stat_to_wire(stat))
|
|
1414
|
+
return index
|
|
1415
|
+
|
|
1416
|
+
units = [return_unit_to_document(unit, encode_present) for unit in outcome.batch.units]
|
|
1417
|
+
return {"batch": {"mode": outcome.batch.mode, "units": units}, "kind": "return"}
|
|
1418
|
+
|
|
1419
|
+
|
|
1420
|
+
def _preflight_value_tree(root: Value, budget: list[int]) -> None:
|
|
1421
|
+
"""Bound recursive ValueCodec work and prove canonical container envelopes."""
|
|
1422
|
+
from dinkster_protocol import RESULT_ALGEBRA_MAX_NESTING
|
|
1423
|
+
|
|
1424
|
+
pending: list[tuple[Value, int]] = [(root, 0)]
|
|
1425
|
+
while pending:
|
|
1426
|
+
value, depth = pending.pop()
|
|
1427
|
+
budget[0] -= 1
|
|
1428
|
+
if budget[0] < 0 or depth > RESULT_ALGEBRA_MAX_NESTING:
|
|
1429
|
+
raise BoundaryError("result algebra value tree is excessive")
|
|
1430
|
+
children = value_children(value)
|
|
1431
|
+
element_type = parse_list_type_id(value.type_id)
|
|
1432
|
+
positions = parse_tuple_type_id(value.type_id)
|
|
1433
|
+
if (children is None) != (element_type is None and positions is None):
|
|
1434
|
+
raise BoundaryError("result algebra value has a noncanonical container envelope")
|
|
1435
|
+
if children is not None:
|
|
1436
|
+
if positions is not None:
|
|
1437
|
+
if not isinstance(value.payload, TuplePayload) or positions != tuple(
|
|
1438
|
+
child.type_id for child in children
|
|
1439
|
+
):
|
|
1440
|
+
raise BoundaryError("result algebra tuple child types do not match positions")
|
|
1441
|
+
elif not isinstance(value.payload, ListPayload) or any(
|
|
1442
|
+
element_type != "*" and child.type_id != element_type for child in children
|
|
1443
|
+
):
|
|
1444
|
+
raise BoundaryError("result algebra list child type does not match its envelope")
|
|
1445
|
+
pending.extend((child, depth + 1) for child in children)
|
|
1446
|
+
|
|
1447
|
+
|
|
1448
|
+
def _finite_number(value: object, name: str) -> float:
|
|
1449
|
+
if type(value) not in (int, float):
|
|
1450
|
+
raise BoundaryError(f"{name} must be a finite number")
|
|
1451
|
+
if type(value) is int and not -(2**63) <= value < 2**63:
|
|
1452
|
+
raise BoundaryError(f"{name} is outside signed 64-bit range")
|
|
1453
|
+
try:
|
|
1454
|
+
converted = float(cast("int | float", value))
|
|
1455
|
+
except (OverflowError, TypeError, ValueError) as exc:
|
|
1456
|
+
raise BoundaryError(f"{name} must be a finite number") from exc
|
|
1457
|
+
if not math.isfinite(converted) or (converted == 0.0 and math.copysign(1.0, converted) < 0):
|
|
1458
|
+
raise BoundaryError(f"{name} must be finite and not negative zero")
|
|
1459
|
+
return converted
|
|
1460
|
+
|
|
1461
|
+
|
|
1462
|
+
def decode_invocation_outcome(
|
|
1463
|
+
codec: ValueCodec,
|
|
1464
|
+
header: Mapping[str, Any],
|
|
1465
|
+
blobs: Sequence[bytes],
|
|
1466
|
+
consumed: list[str],
|
|
1467
|
+
) -> tuple[InvocationOutcome, dict[tuple[int, str], TransferStat]]:
|
|
1468
|
+
"""Strictly decode an invocation return-unit frame."""
|
|
1469
|
+
from dinkster_protocol import (
|
|
1470
|
+
RESULT_ALGEBRA_MAX_COUNT,
|
|
1471
|
+
RESULT_ALGEBRA_MAX_DOCUMENT_BYTES,
|
|
1472
|
+
)
|
|
1473
|
+
|
|
1474
|
+
expected_header = {"type", "invocationId", "executeMs", "resultAlgebra"}
|
|
1475
|
+
if "blobs" in header:
|
|
1476
|
+
expected_header.add("blobs")
|
|
1477
|
+
execute_ms = header.get("executeMs")
|
|
1478
|
+
if (
|
|
1479
|
+
set(header) != expected_header
|
|
1480
|
+
or header.get("type") != "result"
|
|
1481
|
+
or type(header.get("invocationId")) is not str
|
|
1482
|
+
):
|
|
1483
|
+
raise BoundaryError("invalid typed result frame")
|
|
1484
|
+
_finite_number(execute_ms, "typed result executeMs")
|
|
1485
|
+
if "blobs" in header:
|
|
1486
|
+
lengths = header["blobs"]
|
|
1487
|
+
if (
|
|
1488
|
+
type(lengths) is not list
|
|
1489
|
+
or len(lengths) != len(blobs)
|
|
1490
|
+
or any(type(length) is not int or length < 0 for length in lengths)
|
|
1491
|
+
or list(lengths) != [len(blob) for blob in blobs]
|
|
1492
|
+
):
|
|
1493
|
+
raise BoundaryError("typed result frame has invalid blob lengths")
|
|
1494
|
+
algebra = header["resultAlgebra"]
|
|
1495
|
+
if not isinstance(algebra, Mapping) or set(algebra) != {
|
|
1496
|
+
"document",
|
|
1497
|
+
"values",
|
|
1498
|
+
"valueStats",
|
|
1499
|
+
}:
|
|
1500
|
+
raise BoundaryError("invalid resultAlgebra fields")
|
|
1501
|
+
text = algebra["document"]
|
|
1502
|
+
try:
|
|
1503
|
+
if type(text) is not str or len(text.encode("ascii")) > RESULT_ALGEBRA_MAX_DOCUMENT_BYTES:
|
|
1504
|
+
raise BoundaryError("invalid result algebra document")
|
|
1505
|
+
document = json.loads(
|
|
1506
|
+
text,
|
|
1507
|
+
object_pairs_hook=_unique_object,
|
|
1508
|
+
parse_constant=lambda value: (_ for _ in ()).throw(ValueError(value)),
|
|
1509
|
+
)
|
|
1510
|
+
_validate_json_shape(document)
|
|
1511
|
+
except (
|
|
1512
|
+
ValueError,
|
|
1513
|
+
TypeError,
|
|
1514
|
+
json.JSONDecodeError,
|
|
1515
|
+
RecursionError,
|
|
1516
|
+
OverflowError,
|
|
1517
|
+
UnicodeError,
|
|
1518
|
+
) as exc:
|
|
1519
|
+
raise BoundaryError("invalid result algebra JSON") from exc
|
|
1520
|
+
try:
|
|
1521
|
+
canonical = json.dumps(
|
|
1522
|
+
document, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False
|
|
1523
|
+
)
|
|
1524
|
+
except (ValueError, TypeError, OverflowError, UnicodeError, RecursionError) as exc:
|
|
1525
|
+
raise BoundaryError("invalid result algebra JSON") from exc
|
|
1526
|
+
if canonical != text:
|
|
1527
|
+
raise BoundaryError("result algebra document is not canonical")
|
|
1528
|
+
values = algebra["values"]
|
|
1529
|
+
stats = algebra["valueStats"]
|
|
1530
|
+
if (
|
|
1531
|
+
type(values) is not list
|
|
1532
|
+
or type(stats) is not list
|
|
1533
|
+
or len(values) != len(stats)
|
|
1534
|
+
or len(values) > RESULT_ALGEBRA_MAX_COUNT
|
|
1535
|
+
):
|
|
1536
|
+
raise BoundaryError("invalid result algebra value descriptors")
|
|
1537
|
+
descriptor_budget = [RESULT_ALGEBRA_MAX_COUNT]
|
|
1538
|
+
used_blobs: set[int] = set()
|
|
1539
|
+
for index, wire in enumerate(values):
|
|
1540
|
+
if not isinstance(wire, Mapping) or not isinstance(stats[index], Mapping):
|
|
1541
|
+
raise BoundaryError("invalid result algebra value descriptor")
|
|
1542
|
+
_validate_value_descriptor(wire, blobs, used_blobs, budget=descriptor_budget)
|
|
1543
|
+
_typed_stat_from_wire(cast("Mapping[str, Any]", stats[index]))
|
|
1544
|
+
if used_blobs != set(range(len(blobs))):
|
|
1545
|
+
raise BoundaryError("result algebra has unused or duplicate blobs")
|
|
1546
|
+
placeholder = Value(
|
|
1547
|
+
type_id="result.preflight",
|
|
1548
|
+
fingerprint="result.preflight",
|
|
1549
|
+
meta=ValueMeta(),
|
|
1550
|
+
payload=EncodedPayload("result.preflight", b"", None, "inline"),
|
|
1551
|
+
)
|
|
1552
|
+
try:
|
|
1553
|
+
_, preflight_used, _ = _document_to_outcome(document, [placeholder] * len(values))
|
|
1554
|
+
except (KeyError, TypeError, ValueError, RecursionError, OverflowError) as exc:
|
|
1555
|
+
raise BoundaryError("malformed result algebra document") from exc
|
|
1556
|
+
if preflight_used != set(range(len(values))):
|
|
1557
|
+
raise BoundaryError("result algebra has unused or duplicate value descriptors")
|
|
1558
|
+
decoded: list[Value] = []
|
|
1559
|
+
for wire in values:
|
|
1560
|
+
try:
|
|
1561
|
+
value, _ = codec.decode(cast("Mapping[str, Any]", wire), blobs, consumed)
|
|
1562
|
+
except (BoundaryError, KeyError, IndexError, TypeError, ValueError, OverflowError) as exc:
|
|
1563
|
+
raise BoundaryError("invalid result algebra value descriptor") from exc
|
|
1564
|
+
decoded.append(value)
|
|
1565
|
+
try:
|
|
1566
|
+
outcome, used, locations = _document_to_outcome(document, decoded)
|
|
1567
|
+
except (KeyError, TypeError, ValueError, RecursionError, OverflowError) as exc:
|
|
1568
|
+
raise BoundaryError("malformed result algebra document") from exc
|
|
1569
|
+
if used != set(range(len(decoded))):
|
|
1570
|
+
raise BoundaryError("result algebra has unused or duplicate value descriptors")
|
|
1571
|
+
return outcome, {
|
|
1572
|
+
location: _typed_stat_from_wire(cast("Mapping[str, Any]", stats[index]))
|
|
1573
|
+
for index, location in locations.items()
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
|
|
1577
|
+
def _unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]:
|
|
1578
|
+
result: dict[str, object] = {}
|
|
1579
|
+
for key, value in pairs:
|
|
1580
|
+
if key in result:
|
|
1581
|
+
raise ValueError("duplicate JSON field")
|
|
1582
|
+
result[key] = value
|
|
1583
|
+
return result
|
|
1584
|
+
|
|
1585
|
+
|
|
1586
|
+
def _validate_json_shape(value: object) -> None:
|
|
1587
|
+
"""Enforce cumulative canonical-document limits before typed decoding."""
|
|
1588
|
+
from dinkster_protocol import (
|
|
1589
|
+
RESULT_ALGEBRA_MAX_COUNT,
|
|
1590
|
+
RESULT_ALGEBRA_MAX_NESTING,
|
|
1591
|
+
)
|
|
1592
|
+
|
|
1593
|
+
count = 0
|
|
1594
|
+
pending: list[tuple[object, int]] = [(value, 0)]
|
|
1595
|
+
while pending:
|
|
1596
|
+
item, depth = pending.pop()
|
|
1597
|
+
count += 1
|
|
1598
|
+
if count > RESULT_ALGEBRA_MAX_COUNT * 16:
|
|
1599
|
+
raise ValueError("result algebra document has excessive items")
|
|
1600
|
+
if depth > RESULT_ALGEBRA_MAX_NESTING:
|
|
1601
|
+
raise ValueError("result algebra document has excessive nesting")
|
|
1602
|
+
if isinstance(item, dict):
|
|
1603
|
+
pending.extend((child, depth + 1) for child in item.values())
|
|
1604
|
+
elif isinstance(item, list):
|
|
1605
|
+
pending.extend((child, depth + 1) for child in item)
|
|
1606
|
+
elif type(item) is str:
|
|
1607
|
+
item.encode("utf-8")
|
|
1608
|
+
elif type(item) is int:
|
|
1609
|
+
if item < -(2**63) or item > 2**63 - 1:
|
|
1610
|
+
raise ValueError("JSON integer is outside signed 64-bit range")
|
|
1611
|
+
elif type(item) is float:
|
|
1612
|
+
if not math.isfinite(item) or (item == 0.0 and math.copysign(1.0, item) < 0):
|
|
1613
|
+
raise ValueError("JSON float is not canonical and finite")
|
|
1614
|
+
elif item is not None and type(item) is not bool:
|
|
1615
|
+
raise ValueError("unsupported JSON value")
|
|
1616
|
+
|
|
1617
|
+
|
|
1618
|
+
def _blob_index(value: object, blobs: Sequence[bytes], used: set[int]) -> None:
|
|
1619
|
+
if type(value) is not int or value < 0 or value >= len(blobs) or value in used:
|
|
1620
|
+
raise BoundaryError("result algebra has invalid or duplicate blob index")
|
|
1621
|
+
used.add(value)
|
|
1622
|
+
|
|
1623
|
+
|
|
1624
|
+
def _validate_value_descriptor(
|
|
1625
|
+
wire: Mapping[str, Any],
|
|
1626
|
+
blobs: Sequence[bytes],
|
|
1627
|
+
used: set[int],
|
|
1628
|
+
*,
|
|
1629
|
+
budget: list[int],
|
|
1630
|
+
depth: int = 0,
|
|
1631
|
+
expected_type: str | None = None,
|
|
1632
|
+
) -> None:
|
|
1633
|
+
"""Validate the exact ValueCodec wire shape before invoking its permissive decoder."""
|
|
1634
|
+
from dinkster_protocol import RESULT_ALGEBRA_MAX_COUNT, RESULT_ALGEBRA_MAX_NESTING
|
|
1635
|
+
|
|
1636
|
+
budget[0] -= 1
|
|
1637
|
+
if budget[0] < 0 or depth > RESULT_ALGEBRA_MAX_NESTING:
|
|
1638
|
+
raise BoundaryError("result algebra value descriptor nesting is excessive")
|
|
1639
|
+
if (
|
|
1640
|
+
type(wire) is not dict
|
|
1641
|
+
or type(wire.get("typeId")) is not str
|
|
1642
|
+
or type(wire.get("fingerprint")) is not str
|
|
1643
|
+
):
|
|
1644
|
+
raise BoundaryError("invalid result algebra value descriptor")
|
|
1645
|
+
if expected_type is not None and wire["typeId"] != expected_type:
|
|
1646
|
+
raise BoundaryError("result algebra list child type does not match its envelope")
|
|
1647
|
+
_blob_index(wire.get("metaBlob"), blobs, used)
|
|
1648
|
+
element_type = parse_list_type_id(wire["typeId"])
|
|
1649
|
+
positions = parse_tuple_type_id(wire["typeId"])
|
|
1650
|
+
if "elements" in wire:
|
|
1651
|
+
if (element_type is None and positions is None) or set(wire) != {
|
|
1652
|
+
"typeId",
|
|
1653
|
+
"fingerprint",
|
|
1654
|
+
"metaBlob",
|
|
1655
|
+
"elements",
|
|
1656
|
+
}:
|
|
1657
|
+
raise BoundaryError("invalid list value descriptor fields")
|
|
1658
|
+
elements = wire["elements"]
|
|
1659
|
+
if type(elements) is not list or len(elements) > RESULT_ALGEBRA_MAX_COUNT:
|
|
1660
|
+
raise BoundaryError("invalid list value descriptor elements")
|
|
1661
|
+
if positions is not None and len(elements) != len(positions):
|
|
1662
|
+
raise BoundaryError("tuple value descriptor arity does not match its envelope")
|
|
1663
|
+
for index, child in enumerate(elements):
|
|
1664
|
+
if type(child) is not dict:
|
|
1665
|
+
raise BoundaryError("invalid list child value descriptor")
|
|
1666
|
+
_validate_value_descriptor(
|
|
1667
|
+
child,
|
|
1668
|
+
blobs,
|
|
1669
|
+
used,
|
|
1670
|
+
budget=budget,
|
|
1671
|
+
depth=depth + 1,
|
|
1672
|
+
expected_type=(
|
|
1673
|
+
positions[index]
|
|
1674
|
+
if positions is not None
|
|
1675
|
+
else None
|
|
1676
|
+
if element_type == "*"
|
|
1677
|
+
else element_type
|
|
1678
|
+
),
|
|
1679
|
+
)
|
|
1680
|
+
return
|
|
1681
|
+
if (
|
|
1682
|
+
element_type is not None
|
|
1683
|
+
or positions is not None
|
|
1684
|
+
or set(wire) != {"typeId", "fingerprint", "metaBlob", "payload"}
|
|
1685
|
+
):
|
|
1686
|
+
raise BoundaryError("invalid value descriptor fields")
|
|
1687
|
+
payload = wire["payload"]
|
|
1688
|
+
if type(payload) is not dict or type(payload.get("transport")) is not str:
|
|
1689
|
+
raise BoundaryError("invalid value payload descriptor")
|
|
1690
|
+
transport = payload["transport"]
|
|
1691
|
+
if transport == "inline":
|
|
1692
|
+
if set(payload) != {"transport", "blob"}:
|
|
1693
|
+
raise BoundaryError("invalid inline payload descriptor")
|
|
1694
|
+
_blob_index(payload["blob"], blobs, used)
|
|
1695
|
+
elif transport == "shm":
|
|
1696
|
+
if (
|
|
1697
|
+
set(payload) != {"transport", "segment", "size"}
|
|
1698
|
+
or type(payload["segment"]) is not str
|
|
1699
|
+
or type(payload["size"]) is not int
|
|
1700
|
+
or payload["size"] < 0
|
|
1701
|
+
):
|
|
1702
|
+
raise BoundaryError("invalid shared-memory payload descriptor")
|
|
1703
|
+
elif transport == "cas":
|
|
1704
|
+
expected = {"transport", "digest", "size"}
|
|
1705
|
+
if "blob" in payload:
|
|
1706
|
+
expected |= {"blob", "retained"}
|
|
1707
|
+
if (
|
|
1708
|
+
set(payload) != expected
|
|
1709
|
+
or type(payload["digest"]) is not str
|
|
1710
|
+
or type(payload["size"]) is not int
|
|
1711
|
+
or payload["size"] < 0
|
|
1712
|
+
or ("retained" in payload and type(payload["retained"]) is not bool)
|
|
1713
|
+
):
|
|
1714
|
+
raise BoundaryError("invalid CAS payload descriptor")
|
|
1715
|
+
if "blob" in payload:
|
|
1716
|
+
_blob_index(payload["blob"], blobs, used)
|
|
1717
|
+
else:
|
|
1718
|
+
raise BoundaryError("unknown value payload transport")
|
|
1719
|
+
|
|
1720
|
+
|
|
1721
|
+
def _typed_stat_from_wire(wire: Mapping[str, Any]) -> TransferStat:
|
|
1722
|
+
if (
|
|
1723
|
+
type(wire) is not dict
|
|
1724
|
+
or set(wire) != {"transport", "sizeBytes", "codecMs", "declaredCodec", "reused"}
|
|
1725
|
+
or type(wire["transport"]) is not str
|
|
1726
|
+
or type(wire["sizeBytes"]) is not int
|
|
1727
|
+
or wire["sizeBytes"] < 0
|
|
1728
|
+
or type(wire["codecMs"]) not in (int, float)
|
|
1729
|
+
or type(wire["declaredCodec"]) is not bool
|
|
1730
|
+
or type(wire["reused"]) is not bool
|
|
1731
|
+
):
|
|
1732
|
+
raise BoundaryError("invalid result algebra transfer stat")
|
|
1733
|
+
try:
|
|
1734
|
+
codec_ms = _finite_number(wire["codecMs"], "result algebra codecMs")
|
|
1735
|
+
except BoundaryError as exc:
|
|
1736
|
+
raise BoundaryError("invalid result algebra transfer stat") from exc
|
|
1737
|
+
if codec_ms < 0:
|
|
1738
|
+
raise BoundaryError("invalid result algebra transfer stat")
|
|
1739
|
+
return stat_from_wire(wire)
|
|
1740
|
+
|
|
1741
|
+
|
|
1742
|
+
def _exact(value: object, fields: set[str]) -> dict[str, Any]:
|
|
1743
|
+
if not isinstance(value, dict) or set(value) != fields:
|
|
1744
|
+
raise ValueError("unexpected fields")
|
|
1745
|
+
return cast("dict[str, Any]", value)
|
|
1746
|
+
|
|
1747
|
+
|
|
1748
|
+
def _document_to_outcome(
|
|
1749
|
+
document: object, values: list[Value]
|
|
1750
|
+
) -> tuple[InvocationOutcome, set[int], dict[int, tuple[int, str]]]:
|
|
1751
|
+
from dinkster_protocol import (
|
|
1752
|
+
ErrorHint,
|
|
1753
|
+
InvocationFailure,
|
|
1754
|
+
InvocationReturn,
|
|
1755
|
+
NodeError,
|
|
1756
|
+
ReturnBatch,
|
|
1757
|
+
)
|
|
1758
|
+
from dinkster_protocol.result_algebra import ReturnUnit
|
|
1759
|
+
from dinkster_protocol.result_document import return_unit_from_document
|
|
1760
|
+
|
|
1761
|
+
if not isinstance(document, dict):
|
|
1762
|
+
raise ValueError("outcome document must be an object")
|
|
1763
|
+
root = cast("dict[str, Any]", document)
|
|
1764
|
+
if root.get("kind") == "error":
|
|
1765
|
+
root = _exact(root, {"kind", "error"})
|
|
1766
|
+
error = _exact(root["error"], {"hints", "message", "nodeId", "nodeType", "traceback"})
|
|
1767
|
+
if (
|
|
1768
|
+
not all(
|
|
1769
|
+
type(error[key]) is str for key in ("message", "nodeId", "nodeType", "traceback")
|
|
1770
|
+
)
|
|
1771
|
+
or type(error["hints"]) is not list
|
|
1772
|
+
):
|
|
1773
|
+
raise ValueError("invalid error primitives")
|
|
1774
|
+
hints = tuple(
|
|
1775
|
+
ErrorHint(**_exact(item, {"code", "message", "suggestion"})) for item in error["hints"]
|
|
1776
|
+
)
|
|
1777
|
+
if not all(
|
|
1778
|
+
type(h.code) is str
|
|
1779
|
+
and type(h.message) is str
|
|
1780
|
+
and (h.suggestion is None or type(h.suggestion) is str)
|
|
1781
|
+
for h in hints
|
|
1782
|
+
):
|
|
1783
|
+
raise ValueError("invalid hints")
|
|
1784
|
+
return (
|
|
1785
|
+
InvocationFailure(
|
|
1786
|
+
NodeError(
|
|
1787
|
+
error["nodeId"], error["nodeType"], error["message"], error["traceback"], hints
|
|
1788
|
+
)
|
|
1789
|
+
),
|
|
1790
|
+
set(),
|
|
1791
|
+
{},
|
|
1792
|
+
)
|
|
1793
|
+
root = _exact(root, {"kind", "batch"})
|
|
1794
|
+
if root["kind"] != "return":
|
|
1795
|
+
raise ValueError("unknown outcome kind")
|
|
1796
|
+
batch = _exact(root["batch"], {"mode", "units"})
|
|
1797
|
+
used: set[int] = set()
|
|
1798
|
+
locations: dict[int, tuple[int, str]] = {}
|
|
1799
|
+
|
|
1800
|
+
units: list[ReturnUnit] = []
|
|
1801
|
+
if type(batch["units"]) is not list:
|
|
1802
|
+
raise ValueError("units must be an array")
|
|
1803
|
+
for unit_index, raw_unit in enumerate(batch["units"]):
|
|
1804
|
+
|
|
1805
|
+
def decode_present(index: object, output_id: str, unit_index: int = unit_index) -> Value:
|
|
1806
|
+
if type(index) is not int or index < 0 or index >= len(values) or index in used:
|
|
1807
|
+
raise ValueError("invalid descriptor index")
|
|
1808
|
+
used.add(index)
|
|
1809
|
+
locations[index] = (unit_index, output_id)
|
|
1810
|
+
return values[index]
|
|
1811
|
+
|
|
1812
|
+
units.append(return_unit_from_document(raw_unit, decode_present))
|
|
1813
|
+
return InvocationReturn(ReturnBatch(batch["mode"], tuple(units))), used, locations
|
|
1814
|
+
|
|
1815
|
+
|
|
1816
|
+
def decode_result_outputs(
|
|
1817
|
+
codec: ValueCodec, header: Mapping[str, Any], blobs: Sequence[bytes], consumed: list[str]
|
|
1818
|
+
) -> tuple[dict[str, Value], dict[str, TransferStat]]:
|
|
1819
|
+
"""Decode a successful result's outputs. Returned stats merge the
|
|
1820
|
+
sender's encode cost with this side's transfer cost per output."""
|
|
1821
|
+
outputs: dict[str, Value] = {}
|
|
1822
|
+
stats: dict[str, TransferStat] = {}
|
|
1823
|
+
sender_stats = cast("Mapping[str, Any]", header.get("outputStats", {}))
|
|
1824
|
+
for output_id, wire in cast("Mapping[str, Any]", header["outputs"]).items():
|
|
1825
|
+
try:
|
|
1826
|
+
value, local = codec.decode(cast("Mapping[str, Any]", wire), blobs, consumed)
|
|
1827
|
+
except PersistentBlobMissing as exc:
|
|
1828
|
+
raise PersistentBlobMissing(f"output '{output_id}': {exc}") from exc
|
|
1829
|
+
outputs[str(output_id)] = value
|
|
1830
|
+
remote_wire = sender_stats.get(output_id)
|
|
1831
|
+
if remote_wire is not None:
|
|
1832
|
+
remote = stat_from_wire(cast("Mapping[str, Any]", remote_wire))
|
|
1833
|
+
stats[str(output_id)] = TransferStat(
|
|
1834
|
+
transport=remote.transport,
|
|
1835
|
+
size_bytes=remote.size_bytes,
|
|
1836
|
+
codec_ms=remote.codec_ms + local.codec_ms,
|
|
1837
|
+
declared_codec=remote.declared_codec,
|
|
1838
|
+
reused=remote.reused,
|
|
1839
|
+
network_bytes=remote.network_bytes,
|
|
1840
|
+
transfer_ms=remote.transfer_ms,
|
|
1841
|
+
)
|
|
1842
|
+
else:
|
|
1843
|
+
stats[str(output_id)] = local
|
|
1844
|
+
return outputs, stats
|
|
1845
|
+
|
|
1846
|
+
|
|
1847
|
+
def decode_result(
|
|
1848
|
+
codec: ValueCodec,
|
|
1849
|
+
header: Mapping[str, Any],
|
|
1850
|
+
blobs: Sequence[bytes],
|
|
1851
|
+
consumed: list[str],
|
|
1852
|
+
) -> tuple[InvocationResult, dict[str, TransferStat]]:
|
|
1853
|
+
"""Decode ordinary values or one scalar expansion onto the same handoff path."""
|
|
1854
|
+
if "resultAlgebra" not in header:
|
|
1855
|
+
outputs, stats = decode_result_outputs(codec, header, blobs, consumed)
|
|
1856
|
+
return InvocationResult(
|
|
1857
|
+
outputs=outputs,
|
|
1858
|
+
preview_artifacts=_decode_preview_artifacts(header, blobs),
|
|
1859
|
+
saved_artifact_claims=decode_saved_artifact_claims(header),
|
|
1860
|
+
saved_file_previews=decode_saved_file_previews(header),
|
|
1861
|
+
), stats
|
|
1862
|
+
from dinkster_protocol import ExpandedReturn, InvocationReturn
|
|
1863
|
+
|
|
1864
|
+
# Conversation identity and artifact claims are validated by the session;
|
|
1865
|
+
# the algebra codec owns only the result payload's exact envelope.
|
|
1866
|
+
fields = {"type", "invocationId", "executeMs", "resultAlgebra", "blobs"}
|
|
1867
|
+
if (
|
|
1868
|
+
set(header)
|
|
1869
|
+
- fields
|
|
1870
|
+
- {
|
|
1871
|
+
"jobRef",
|
|
1872
|
+
"attemptId",
|
|
1873
|
+
"artifactCandidates",
|
|
1874
|
+
"previewArtifacts",
|
|
1875
|
+
"savedArtifactClaims",
|
|
1876
|
+
"savedFilePreviews",
|
|
1877
|
+
}
|
|
1878
|
+
):
|
|
1879
|
+
raise BoundaryError("invalid invocation expansion frame fields")
|
|
1880
|
+
payload = {key: header[key] for key in fields if key in header}
|
|
1881
|
+
previews = _decode_preview_artifacts(header, blobs)
|
|
1882
|
+
value_blobs = blobs[: len(blobs) - len(previews)]
|
|
1883
|
+
if "blobs" in payload:
|
|
1884
|
+
lengths = payload["blobs"]
|
|
1885
|
+
if type(lengths) is not list or lengths != [len(blob) for blob in blobs]:
|
|
1886
|
+
raise BoundaryError("preview result has invalid blob lengths")
|
|
1887
|
+
payload["blobs"] = lengths[: len(value_blobs)]
|
|
1888
|
+
outcome, stats = decode_invocation_outcome(codec, payload, value_blobs, consumed)
|
|
1889
|
+
if (
|
|
1890
|
+
not isinstance(outcome, InvocationReturn)
|
|
1891
|
+
or outcome.batch.mode != "scalar"
|
|
1892
|
+
or not isinstance(outcome.batch.units[0], ExpandedReturn)
|
|
1893
|
+
):
|
|
1894
|
+
raise BoundaryError("invocation expansion must contain one scalar expanded return")
|
|
1895
|
+
return InvocationResult(
|
|
1896
|
+
expansion=outcome.batch.units[0],
|
|
1897
|
+
preview_artifacts=previews,
|
|
1898
|
+
saved_artifact_claims=decode_saved_artifact_claims(header),
|
|
1899
|
+
saved_file_previews=decode_saved_file_previews(header),
|
|
1900
|
+
), {name: stat for (_, name), stat in stats.items()}
|
|
1901
|
+
|
|
1902
|
+
|
|
1903
|
+
def error_from_wire(wire: Mapping[str, Any]) -> NodeError:
|
|
1904
|
+
hints = tuple(
|
|
1905
|
+
ErrorHint(
|
|
1906
|
+
code=str(hint.get("code", "")),
|
|
1907
|
+
message=str(hint.get("message", "")),
|
|
1908
|
+
suggestion=(str(hint["suggestion"]) if hint.get("suggestion") is not None else None),
|
|
1909
|
+
)
|
|
1910
|
+
for hint in cast("Sequence[Mapping[str, Any]]", wire.get("hints", ()))
|
|
1911
|
+
)
|
|
1912
|
+
return NodeError(
|
|
1913
|
+
node_id=str(wire.get("nodeId", "")),
|
|
1914
|
+
node_type=str(wire.get("nodeType", "")),
|
|
1915
|
+
message=str(wire.get("message", "")),
|
|
1916
|
+
traceback=str(wire.get("traceback", "")),
|
|
1917
|
+
hints=hints,
|
|
1918
|
+
)
|