onec-interactive-runtime-core 0.1.19__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- onec_interactive_runtime_core-0.1.19.dist-info/METADATA +12 -0
- onec_interactive_runtime_core-0.1.19.dist-info/RECORD +87 -0
- onec_interactive_runtime_core-0.1.19.dist-info/WHEEL +4 -0
- onec_interactive_runtime_core-0.1.19.dist-info/licenses/COPYRIGHT +9 -0
- onec_interactive_runtime_core-0.1.19.dist-info/licenses/LICENSE +232 -0
- onec_interactive_runtime_core-0.1.19.dist-info/licenses/THIRD_PARTY_NOTICES.txt +32 -0
- onec_runtime/AGENTS.md +7 -0
- onec_runtime/__init__.py +3 -0
- onec_runtime/artifacts.py +59 -0
- onec_runtime/bootstrap.py +310 -0
- onec_runtime/breakpoint_workspace.py +249 -0
- onec_runtime/bsl/__init__.py +139 -0
- onec_runtime/bsl/diagnostics.py +1046 -0
- onec_runtime/bsl/full_ast_worker_projection.py +522 -0
- onec_runtime/bsl/generated_semantic_parser.py +6322 -0
- onec_runtime/bsl/lexer.py +164 -0
- onec_runtime/bsl/module_catalog.py +420 -0
- onec_runtime/bsl/module_delta.py +1079 -0
- onec_runtime/bsl/module_universe.py +1877 -0
- onec_runtime/bsl/notebook_cells.py +232 -0
- onec_runtime/bsl/notebook_method_globals.py +74 -0
- onec_runtime/bsl/notebook_methods.py +175 -0
- onec_runtime/bsl/parser_artifact_identity.py +119 -0
- onec_runtime/bsl/parser_target.py +388 -0
- onec_runtime/bsl/preprocessor.py +16 -0
- onec_runtime/bsl/semantic_lowering.py +1539 -0
- onec_runtime/bsl/source_maps.py +2411 -0
- onec_runtime/bsl/worker_dependency_resolver.py +289 -0
- onec_runtime/bsl/worker_preprocessor.py +247 -0
- onec_runtime/bsl/worker_projection_model.py +137 -0
- onec_runtime/bsl/worker_reload_source_map.py +1697 -0
- onec_runtime/capture.py +113 -0
- onec_runtime/capture_source.py +453 -0
- onec_runtime/compact_table.py +376 -0
- onec_runtime/compact_table_backend.py +500 -0
- onec_runtime/config.py +198 -0
- onec_runtime/configurator_agent.py +520 -0
- onec_runtime/controller_worker.py +400 -0
- onec_runtime/credentials.py +18 -0
- onec_runtime/epf_container.py +196 -0
- onec_runtime/errors.py +205 -0
- onec_runtime/experiment.py +130 -0
- onec_runtime/extension_bundle.py +944 -0
- onec_runtime/extension_lifecycle.py +402 -0
- onec_runtime/extension_state.py +222 -0
- onec_runtime/fault_injection.py +30 -0
- onec_runtime/kernel.py +179 -0
- onec_runtime/lease.py +123 -0
- onec_runtime/observation.py +287 -0
- onec_runtime/performance_profile.py +245 -0
- onec_runtime/privacy.py +263 -0
- onec_runtime/processes.py +265 -0
- onec_runtime/prototype_runtime.py +2644 -0
- onec_runtime/rdbg/__init__.py +1 -0
- onec_runtime/rdbg/models.py +130 -0
- onec_runtime/rdbg/reconnect.py +59 -0
- onec_runtime/rdbg/session.py +913 -0
- onec_runtime/rdbg/transport.py +138 -0
- onec_runtime/rdbg/xml_codec.py +980 -0
- onec_runtime/recovery.py +106 -0
- onec_runtime/recovery_journal.py +79 -0
- onec_runtime/resources/extension/OnecInteractiveRuntime.cfe +0 -0
- onec_runtime/resources/extension/extension-manifest.json +108 -0
- onec_runtime/runtime_api.py +5272 -0
- onec_runtime/runtime_contracts.py +196 -0
- onec_runtime/server_worker.py +2034 -0
- onec_runtime/session.py +2123 -0
- onec_runtime/startup_diagnostics.py +45 -0
- onec_runtime/stop_routing.py +65 -0
- onec_runtime/supervised_processes.py +232 -0
- onec_runtime/supervised_runtime.py +512 -0
- onec_runtime/supervisor.py +363 -0
- onec_runtime/supervisor_model.py +148 -0
- onec_runtime/supervisor_protocol.py +77 -0
- onec_runtime/table_materialization.py +471 -0
- onec_runtime/table_transfer_backend.py +203 -0
- onec_runtime/table_value.py +311 -0
- onec_runtime/toolchain.py +730 -0
- onec_runtime/value_materialization.py +432 -0
- onec_runtime/value_transfer_backend.py +199 -0
- onec_runtime/worker_breakpoints.py +1431 -0
- onec_runtime/worker_epf.py +313 -0
- onec_runtime/worker_epf_template.py +53 -0
- onec_runtime/worker_stage_protocol.py +516 -0
- onec_runtime/worker_universe.py +3772 -0
- onec_runtime_build/__init__.py +1 -0
- onec_runtime_build/parser_target_development.py +116 -0
|
@@ -0,0 +1,2034 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from base64 import b64encode
|
|
4
|
+
from bisect import bisect_right
|
|
5
|
+
from dataclasses import dataclass, replace
|
|
6
|
+
from enum import Enum
|
|
7
|
+
from hashlib import sha256
|
|
8
|
+
from hmac import compare_digest, digest
|
|
9
|
+
import json
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from re import fullmatch, search
|
|
12
|
+
from secrets import token_bytes
|
|
13
|
+
from types import MappingProxyType
|
|
14
|
+
from typing import Literal, cast
|
|
15
|
+
from weakref import ReferenceType, WeakKeyDictionary, ref
|
|
16
|
+
|
|
17
|
+
from onec_runtime.config import RuntimeConfig
|
|
18
|
+
from onec_runtime.errors import (
|
|
19
|
+
BslExecutionError,
|
|
20
|
+
ProtocolError,
|
|
21
|
+
)
|
|
22
|
+
from onec_runtime.experiment import bsl_string_literal
|
|
23
|
+
from onec_runtime.worker_stage_protocol import (
|
|
24
|
+
_safe_identifier as _worker_stage_safe_identifier,
|
|
25
|
+
)
|
|
26
|
+
from onec_runtime.bsl import (
|
|
27
|
+
DiagnosticStage,
|
|
28
|
+
MappedSource,
|
|
29
|
+
SemanticNotebookLowerer,
|
|
30
|
+
SourceArtifactKind,
|
|
31
|
+
SourceUnitRef,
|
|
32
|
+
VisibleSourceContext,
|
|
33
|
+
parse_platform_diagnostic,
|
|
34
|
+
remap_platform_diagnostic,
|
|
35
|
+
)
|
|
36
|
+
from onec_runtime.bsl.diagnostics import (
|
|
37
|
+
WorkerDiagnosticArtifact,
|
|
38
|
+
remap_worker_stage_diagnostic,
|
|
39
|
+
)
|
|
40
|
+
from onec_runtime.bsl.parser_target import PythonParserTarget
|
|
41
|
+
from onec_runtime.bsl.source_maps import (
|
|
42
|
+
SourceArtifactRef,
|
|
43
|
+
SourceMap,
|
|
44
|
+
SourceMapSegment,
|
|
45
|
+
SourceSpan,
|
|
46
|
+
SourceUnitRef,
|
|
47
|
+
)
|
|
48
|
+
from onec_runtime.bsl.module_universe import (
|
|
49
|
+
WorkerSemanticAdmission,
|
|
50
|
+
_validate_worker_semantic_admission,
|
|
51
|
+
)
|
|
52
|
+
from onec_runtime.bsl.semantic_lowering import WorkerExport
|
|
53
|
+
from onec_runtime.bsl.worker_reload_source_map import (
|
|
54
|
+
CompactMappedWorkerSource,
|
|
55
|
+
CompactReloadSourceMap,
|
|
56
|
+
snapshot_compact_mapped_worker_source,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(frozen=True, slots=True)
|
|
61
|
+
class WorkerSourceProvenance:
|
|
62
|
+
source_sha256: str
|
|
63
|
+
source_map_sha256: str
|
|
64
|
+
worker_generation: int | None = None
|
|
65
|
+
worker_manifest_sha256: str | None = None
|
|
66
|
+
|
|
67
|
+
def __post_init__(self) -> None:
|
|
68
|
+
for name in ("source_sha256", "source_map_sha256"):
|
|
69
|
+
if fullmatch(r"[0-9a-f]{64}", getattr(self, name)) is None:
|
|
70
|
+
raise ValueError(f"{name} must be a lowercase sha256")
|
|
71
|
+
if (
|
|
72
|
+
self.worker_generation is not None
|
|
73
|
+
and (
|
|
74
|
+
type(self.worker_generation) is not int
|
|
75
|
+
or self.worker_generation < 0
|
|
76
|
+
)
|
|
77
|
+
):
|
|
78
|
+
raise ValueError("worker_generation must be a nonnegative integer")
|
|
79
|
+
if (
|
|
80
|
+
self.worker_manifest_sha256 is not None
|
|
81
|
+
and fullmatch(r"[0-9a-f]{64}", self.worker_manifest_sha256) is None
|
|
82
|
+
):
|
|
83
|
+
raise ValueError("worker_manifest_sha256 must be a lowercase sha256")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass(frozen=True, slots=True, weakref_slot=True, eq=False)
|
|
87
|
+
class WorkerArtifact:
|
|
88
|
+
logical_name: str
|
|
89
|
+
source_sha256: str
|
|
90
|
+
artifact_sha256: str
|
|
91
|
+
exports: tuple[WorkerExport, ...] = ()
|
|
92
|
+
source_provenance: WorkerSourceProvenance | None = None
|
|
93
|
+
|
|
94
|
+
@property
|
|
95
|
+
def source_map_sha256(self) -> str | None:
|
|
96
|
+
return (
|
|
97
|
+
None
|
|
98
|
+
if self.source_provenance is None
|
|
99
|
+
else self.source_provenance.source_map_sha256
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def _admission(self) -> _WorkerArtifactAdmission | None:
|
|
104
|
+
capability = _worker_artifact_capability(self, required=False)
|
|
105
|
+
return None if capability is None else capability.admission
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
_WORKER_ARTIFACT_STAGE_INSTRUCTIONS: WeakKeyDictionary[
|
|
109
|
+
object,
|
|
110
|
+
str,
|
|
111
|
+
] = WeakKeyDictionary()
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@dataclass(frozen=True, slots=True, repr=False, eq=False, weakref_slot=True, init=False)
|
|
115
|
+
class WorkerArtifactStage:
|
|
116
|
+
"""One content-addressed external-processing connect operation."""
|
|
117
|
+
|
|
118
|
+
logical_name: str
|
|
119
|
+
artifact_sha256: str
|
|
120
|
+
registration_name: str
|
|
121
|
+
phase: Literal["connect"]
|
|
122
|
+
|
|
123
|
+
def __init__(
|
|
124
|
+
self,
|
|
125
|
+
logical_name: str,
|
|
126
|
+
artifact_sha256: str,
|
|
127
|
+
registration_name: str,
|
|
128
|
+
phase: Literal["connect"],
|
|
129
|
+
instruction: str,
|
|
130
|
+
) -> None:
|
|
131
|
+
if (
|
|
132
|
+
not _worker_safe_identifier(logical_name)
|
|
133
|
+
or fullmatch(r"[0-9a-f]{64}", artifact_sha256) is None
|
|
134
|
+
or not _worker_safe_identifier(registration_name)
|
|
135
|
+
or phase != "connect"
|
|
136
|
+
or not isinstance(instruction, str)
|
|
137
|
+
or not instruction
|
|
138
|
+
):
|
|
139
|
+
raise ValueError("worker artifact stage is invalid")
|
|
140
|
+
object.__setattr__(self, "logical_name", logical_name)
|
|
141
|
+
object.__setattr__(self, "artifact_sha256", artifact_sha256)
|
|
142
|
+
object.__setattr__(self, "registration_name", registration_name)
|
|
143
|
+
object.__setattr__(self, "phase", phase)
|
|
144
|
+
_WORKER_ARTIFACT_STAGE_INSTRUCTIONS[self] = instruction
|
|
145
|
+
|
|
146
|
+
@property
|
|
147
|
+
def instruction(self) -> str:
|
|
148
|
+
instruction = _WORKER_ARTIFACT_STAGE_INSTRUCTIONS.get(self)
|
|
149
|
+
if instruction is None:
|
|
150
|
+
raise ProtocolError("Worker artifact stage capability is unavailable")
|
|
151
|
+
return instruction
|
|
152
|
+
|
|
153
|
+
def __deepcopy__(self, memo: dict[int, object]) -> WorkerArtifactStage:
|
|
154
|
+
del memo
|
|
155
|
+
return self
|
|
156
|
+
|
|
157
|
+
def __repr__(self) -> str:
|
|
158
|
+
return (
|
|
159
|
+
"WorkerArtifactStage("
|
|
160
|
+
f"logical_name={self.logical_name!r}, "
|
|
161
|
+
f"artifact_sha256={self.artifact_sha256!r}, "
|
|
162
|
+
f"registration_name={self.registration_name!r}, "
|
|
163
|
+
f"phase={self.phase!r}, instruction=<redacted>)"
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
@dataclass(frozen=True, slots=True)
|
|
168
|
+
class WorkerArtifactStageFailure:
|
|
169
|
+
artifact_sha256: str
|
|
170
|
+
phase: Literal["decode", "upload", "connect", "create"]
|
|
171
|
+
|
|
172
|
+
def __post_init__(self) -> None:
|
|
173
|
+
if (
|
|
174
|
+
fullmatch(r"[0-9a-f]{64}", self.artifact_sha256) is None
|
|
175
|
+
or self.phase not in {"decode", "upload", "connect", "create"}
|
|
176
|
+
):
|
|
177
|
+
raise ValueError("worker artifact stage failure identity is invalid")
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
@dataclass(frozen=True, slots=True, repr=False)
|
|
181
|
+
class _AdmittedWorkerBinary:
|
|
182
|
+
source_bytes: bytes
|
|
183
|
+
artifact_bytes: bytes
|
|
184
|
+
source_sha256: str
|
|
185
|
+
artifact_sha256: str
|
|
186
|
+
|
|
187
|
+
def __repr__(self) -> str:
|
|
188
|
+
return "<redacted admitted worker binary>"
|
|
189
|
+
|
|
190
|
+
def __deepcopy__(self, memo: dict[int, object]) -> _AdmittedWorkerBinary:
|
|
191
|
+
del memo
|
|
192
|
+
return self
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
@dataclass(frozen=True, slots=True, repr=False)
|
|
196
|
+
class _AdmittedWorkerSnapshot:
|
|
197
|
+
binary: _AdmittedWorkerBinary
|
|
198
|
+
proof: bytes
|
|
199
|
+
mapped_source: MappedSource
|
|
200
|
+
mapped_guard: tuple[int, ...]
|
|
201
|
+
source_map_sha256: str
|
|
202
|
+
worker_generation: int | None
|
|
203
|
+
worker_manifest_sha256: str | None
|
|
204
|
+
catalog: tuple[WorkerExport, ...]
|
|
205
|
+
visible_context_snapshot: _WorkerVisibleContextSnapshot | None
|
|
206
|
+
visible_context_fingerprint: str
|
|
207
|
+
visible_context_guard: tuple[object, ...]
|
|
208
|
+
source_path: Path
|
|
209
|
+
artifact_path: Path
|
|
210
|
+
expected_version: str | None
|
|
211
|
+
expected_value: int | None
|
|
212
|
+
|
|
213
|
+
@property
|
|
214
|
+
def source_bytes(self) -> bytes:
|
|
215
|
+
return self.binary.source_bytes
|
|
216
|
+
|
|
217
|
+
@property
|
|
218
|
+
def artifact_bytes(self) -> bytes:
|
|
219
|
+
return self.binary.artifact_bytes
|
|
220
|
+
|
|
221
|
+
@property
|
|
222
|
+
def source_sha256(self) -> str:
|
|
223
|
+
return self.binary.source_sha256
|
|
224
|
+
|
|
225
|
+
@property
|
|
226
|
+
def artifact_sha256(self) -> str:
|
|
227
|
+
return self.binary.artifact_sha256
|
|
228
|
+
|
|
229
|
+
def __repr__(self) -> str:
|
|
230
|
+
return "<redacted admitted worker snapshot>"
|
|
231
|
+
|
|
232
|
+
def __deepcopy__(self, memo: dict[int, object]) -> _AdmittedWorkerSnapshot:
|
|
233
|
+
del memo
|
|
234
|
+
return self
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
class _WorkerArtifactAdmission:
|
|
238
|
+
"""Weak owner capability over one immutable admitted snapshot."""
|
|
239
|
+
|
|
240
|
+
__slots__ = ("__owner", "__snapshot")
|
|
241
|
+
|
|
242
|
+
def __init__(
|
|
243
|
+
self,
|
|
244
|
+
owner: WorkerArtifact,
|
|
245
|
+
snapshot: _AdmittedWorkerSnapshot,
|
|
246
|
+
) -> None:
|
|
247
|
+
object.__setattr__(self, "_WorkerArtifactAdmission__owner", ref(owner))
|
|
248
|
+
object.__setattr__(self, "_WorkerArtifactAdmission__snapshot", snapshot)
|
|
249
|
+
|
|
250
|
+
def __setattr__(self, name: str, value: object) -> None:
|
|
251
|
+
raise AttributeError("worker artifact admission snapshots are immutable")
|
|
252
|
+
|
|
253
|
+
def __repr__(self) -> str:
|
|
254
|
+
return "<redacted worker artifact admission>"
|
|
255
|
+
|
|
256
|
+
def __deepcopy__(self, memo: dict[int, object]) -> str:
|
|
257
|
+
return "<redacted worker artifact admission>"
|
|
258
|
+
|
|
259
|
+
def snapshot(self, owner: WorkerArtifact) -> _AdmittedWorkerSnapshot:
|
|
260
|
+
owner_ref: ReferenceType[WorkerArtifact] = self.__owner
|
|
261
|
+
if owner_ref() is not owner:
|
|
262
|
+
raise ProtocolError("Runtime API requires a proven worker artifact")
|
|
263
|
+
return self.__snapshot
|
|
264
|
+
|
|
265
|
+
def contents(
|
|
266
|
+
self,
|
|
267
|
+
owner: WorkerArtifact,
|
|
268
|
+
) -> tuple[bytes, bytes, bytes, MappedSource]:
|
|
269
|
+
snapshot = self.snapshot(owner)
|
|
270
|
+
return (
|
|
271
|
+
snapshot.source_bytes,
|
|
272
|
+
snapshot.artifact_bytes,
|
|
273
|
+
snapshot.proof,
|
|
274
|
+
_snapshot_mapped_source(snapshot.mapped_source)[0],
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
def diagnostic_context(
|
|
278
|
+
self,
|
|
279
|
+
owner: WorkerArtifact,
|
|
280
|
+
) -> VisibleSourceContext | None:
|
|
281
|
+
snapshot = _clone_visible_context_snapshot(
|
|
282
|
+
self.snapshot(owner).visible_context_snapshot
|
|
283
|
+
)
|
|
284
|
+
return None if snapshot is None else snapshot.context
|
|
285
|
+
|
|
286
|
+
def context_snapshot(
|
|
287
|
+
self,
|
|
288
|
+
owner: WorkerArtifact,
|
|
289
|
+
) -> _WorkerVisibleContextSnapshot | None:
|
|
290
|
+
return _clone_visible_context_snapshot(
|
|
291
|
+
self.snapshot(owner).visible_context_snapshot
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
def catalog(self, owner: WorkerArtifact) -> tuple[WorkerExport, ...]:
|
|
295
|
+
catalog = self.snapshot(owner).catalog
|
|
296
|
+
if catalog != owner.exports:
|
|
297
|
+
raise ProtocolError("Runtime API requires a proven worker artifact")
|
|
298
|
+
return catalog
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
class _ImmutableLineIndex:
|
|
302
|
+
__slots__ = ("_length", "_line_starts")
|
|
303
|
+
|
|
304
|
+
def __init__(self, length: int, line_starts: tuple[int, ...]) -> None:
|
|
305
|
+
if (
|
|
306
|
+
type(length) is not int
|
|
307
|
+
or length < 0
|
|
308
|
+
or not line_starts
|
|
309
|
+
or line_starts[0] != 0
|
|
310
|
+
or any(type(start) is not int for start in line_starts)
|
|
311
|
+
or any(
|
|
312
|
+
current >= following
|
|
313
|
+
for current, following in zip(line_starts, line_starts[1:])
|
|
314
|
+
)
|
|
315
|
+
or line_starts[-1] > length
|
|
316
|
+
):
|
|
317
|
+
raise ProtocolError("Worker visible source line index is invalid")
|
|
318
|
+
object.__setattr__(self, "_length", length)
|
|
319
|
+
object.__setattr__(self, "_line_starts", line_starts)
|
|
320
|
+
|
|
321
|
+
def offset_to_line_column(self, offset: int) -> tuple[int, int]:
|
|
322
|
+
if type(offset) is not int or not 0 <= offset <= self._length:
|
|
323
|
+
raise ValueError("offset must be within the source")
|
|
324
|
+
line_index = bisect_right(self._line_starts, offset) - 1
|
|
325
|
+
return line_index + 1, offset - self._line_starts[line_index] + 1
|
|
326
|
+
|
|
327
|
+
def line_range(self, line: int) -> SourceSpan:
|
|
328
|
+
if type(line) is not int or not 1 <= line <= len(self._line_starts):
|
|
329
|
+
raise ValueError("line must be within the source")
|
|
330
|
+
start = self._line_starts[line - 1]
|
|
331
|
+
end = (
|
|
332
|
+
self._line_starts[line]
|
|
333
|
+
if line < len(self._line_starts)
|
|
334
|
+
else self._length
|
|
335
|
+
)
|
|
336
|
+
return SourceSpan(start, end)
|
|
337
|
+
|
|
338
|
+
def __setattr__(self, name: str, value: object) -> None:
|
|
339
|
+
raise AttributeError("Worker visible line indexes are immutable")
|
|
340
|
+
|
|
341
|
+
def __delattr__(self, name: str) -> None:
|
|
342
|
+
raise AttributeError("Worker visible line indexes are immutable")
|
|
343
|
+
|
|
344
|
+
def __repr__(self) -> str:
|
|
345
|
+
return "<redacted worker visible line index>"
|
|
346
|
+
|
|
347
|
+
def __deepcopy__(self, memo: dict[int, object]) -> _ImmutableLineIndex:
|
|
348
|
+
return self
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
class _ImmutableVisibleSourceContext(VisibleSourceContext):
|
|
352
|
+
__slots__ = ()
|
|
353
|
+
|
|
354
|
+
def __init__(
|
|
355
|
+
self,
|
|
356
|
+
indices: dict[
|
|
357
|
+
tuple[object, str, int],
|
|
358
|
+
tuple[str, _ImmutableLineIndex],
|
|
359
|
+
],
|
|
360
|
+
) -> None:
|
|
361
|
+
object.__setattr__(self, "_indices", MappingProxyType(dict(indices)))
|
|
362
|
+
|
|
363
|
+
def __setattr__(self, name: str, value: object) -> None:
|
|
364
|
+
raise AttributeError("Worker visible source contexts are immutable")
|
|
365
|
+
|
|
366
|
+
def __delattr__(self, name: str) -> None:
|
|
367
|
+
raise AttributeError("Worker visible source contexts are immutable")
|
|
368
|
+
|
|
369
|
+
def __deepcopy__(
|
|
370
|
+
self,
|
|
371
|
+
memo: dict[int, object],
|
|
372
|
+
) -> _ImmutableVisibleSourceContext:
|
|
373
|
+
return self
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
class _WorkerVisibleContextSnapshot:
|
|
377
|
+
__slots__ = ("context", "fingerprint")
|
|
378
|
+
|
|
379
|
+
def __init__(
|
|
380
|
+
self,
|
|
381
|
+
context: _ImmutableVisibleSourceContext,
|
|
382
|
+
fingerprint: str,
|
|
383
|
+
) -> None:
|
|
384
|
+
object.__setattr__(self, "context", context)
|
|
385
|
+
object.__setattr__(self, "fingerprint", fingerprint)
|
|
386
|
+
|
|
387
|
+
def __setattr__(self, name: str, value: object) -> None:
|
|
388
|
+
raise AttributeError("Worker visible context snapshots are immutable")
|
|
389
|
+
|
|
390
|
+
def __repr__(self) -> str:
|
|
391
|
+
return "<redacted worker visible context snapshot>"
|
|
392
|
+
|
|
393
|
+
def __deepcopy__(
|
|
394
|
+
self,
|
|
395
|
+
memo: dict[int, object],
|
|
396
|
+
) -> _WorkerVisibleContextSnapshot:
|
|
397
|
+
return self
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _clone_visible_context_snapshot(
|
|
401
|
+
snapshot: _WorkerVisibleContextSnapshot | None,
|
|
402
|
+
) -> _WorkerVisibleContextSnapshot | None:
|
|
403
|
+
if snapshot is None:
|
|
404
|
+
return None
|
|
405
|
+
try:
|
|
406
|
+
source_indices = snapshot.context._indices # type: ignore[attr-defined]
|
|
407
|
+
indices: dict[
|
|
408
|
+
tuple[object, str, int],
|
|
409
|
+
tuple[str, _ImmutableLineIndex],
|
|
410
|
+
] = {}
|
|
411
|
+
for key, (source_hash, line_index) in source_indices.items():
|
|
412
|
+
indices[key] = (
|
|
413
|
+
source_hash,
|
|
414
|
+
_ImmutableLineIndex(
|
|
415
|
+
line_index._length, # type: ignore[attr-defined]
|
|
416
|
+
line_index._line_starts, # type: ignore[attr-defined]
|
|
417
|
+
),
|
|
418
|
+
)
|
|
419
|
+
return _WorkerVisibleContextSnapshot(
|
|
420
|
+
_ImmutableVisibleSourceContext(indices),
|
|
421
|
+
snapshot.fingerprint,
|
|
422
|
+
)
|
|
423
|
+
except Exception:
|
|
424
|
+
raise ProtocolError(
|
|
425
|
+
"Worker visible source context fingerprint does not match"
|
|
426
|
+
) from None
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
class _WorkerArtifactCapability:
|
|
430
|
+
__slots__ = (
|
|
431
|
+
"artifact_path",
|
|
432
|
+
"source_path",
|
|
433
|
+
"expected_version",
|
|
434
|
+
"expected_value",
|
|
435
|
+
"admission",
|
|
436
|
+
)
|
|
437
|
+
|
|
438
|
+
def __init__(
|
|
439
|
+
self,
|
|
440
|
+
*,
|
|
441
|
+
artifact_path: Path | None,
|
|
442
|
+
source_path: Path | None,
|
|
443
|
+
expected_version: str | None,
|
|
444
|
+
expected_value: int | None,
|
|
445
|
+
admission: _WorkerArtifactAdmission | None,
|
|
446
|
+
) -> None:
|
|
447
|
+
object.__setattr__(self, "artifact_path", artifact_path)
|
|
448
|
+
object.__setattr__(self, "source_path", source_path)
|
|
449
|
+
object.__setattr__(self, "expected_version", expected_version)
|
|
450
|
+
object.__setattr__(self, "expected_value", expected_value)
|
|
451
|
+
object.__setattr__(self, "admission", admission)
|
|
452
|
+
|
|
453
|
+
def __setattr__(self, name: str, value: object) -> None:
|
|
454
|
+
raise AttributeError("Worker artifact capabilities are immutable")
|
|
455
|
+
|
|
456
|
+
def __repr__(self) -> str:
|
|
457
|
+
return "<redacted worker artifact capability>"
|
|
458
|
+
|
|
459
|
+
def __deepcopy__(self, memo: dict[int, object]) -> str:
|
|
460
|
+
return "<redacted worker artifact capability>"
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
_WORKER_ARTIFACT_CAPABILITIES: WeakKeyDictionary[
|
|
464
|
+
WorkerArtifact,
|
|
465
|
+
_WorkerArtifactCapability,
|
|
466
|
+
] = WeakKeyDictionary()
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _bind_worker_artifact_capability(
|
|
470
|
+
artifact: WorkerArtifact,
|
|
471
|
+
*,
|
|
472
|
+
artifact_path: Path | None,
|
|
473
|
+
source_path: Path | None,
|
|
474
|
+
expected_version: str | None,
|
|
475
|
+
expected_value: int | None,
|
|
476
|
+
admission: _WorkerArtifactAdmission | None,
|
|
477
|
+
) -> None:
|
|
478
|
+
_WORKER_ARTIFACT_CAPABILITIES[artifact] = _WorkerArtifactCapability(
|
|
479
|
+
artifact_path=artifact_path,
|
|
480
|
+
source_path=source_path,
|
|
481
|
+
expected_version=expected_version,
|
|
482
|
+
expected_value=expected_value,
|
|
483
|
+
admission=admission,
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _worker_artifact_capability(
|
|
488
|
+
artifact: WorkerArtifact,
|
|
489
|
+
*,
|
|
490
|
+
required: bool = True,
|
|
491
|
+
) -> _WorkerArtifactCapability | None:
|
|
492
|
+
capability = _WORKER_ARTIFACT_CAPABILITIES.get(artifact)
|
|
493
|
+
if capability is None and required:
|
|
494
|
+
raise ProtocolError("Runtime API requires a proven worker artifact")
|
|
495
|
+
return capability
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
_ARTIFACT_PROOF_KEY = token_bytes(32)
|
|
499
|
+
_SOURCE_CATALOG_PROOF_KEY = token_bytes(32)
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def _span_structure(span: SourceSpan | None) -> tuple[int, int] | None:
|
|
503
|
+
return None if span is None else (span.start, span.end)
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def _reference_structure(
|
|
507
|
+
reference: SourceUnitRef | SourceArtifactRef | None,
|
|
508
|
+
) -> tuple[object, ...] | None:
|
|
509
|
+
if reference is None:
|
|
510
|
+
return None
|
|
511
|
+
if isinstance(reference, SourceUnitRef):
|
|
512
|
+
return (
|
|
513
|
+
"unit",
|
|
514
|
+
reference.kind.value,
|
|
515
|
+
reference.unit_id,
|
|
516
|
+
reference.revision,
|
|
517
|
+
reference.source_sha256,
|
|
518
|
+
)
|
|
519
|
+
return (
|
|
520
|
+
"artifact",
|
|
521
|
+
reference.kind.value,
|
|
522
|
+
reference.source_sha256,
|
|
523
|
+
reference.character_length,
|
|
524
|
+
reference.line_ending_kind,
|
|
525
|
+
reference.lowering_semantic_version,
|
|
526
|
+
reference.wrapper_semantic_version,
|
|
527
|
+
reference.mode,
|
|
528
|
+
reference.worker_generation,
|
|
529
|
+
reference.worker_manifest_sha256,
|
|
530
|
+
reference.export_catalog_sha256,
|
|
531
|
+
)
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
def _source_map_structure(source_map: SourceMap) -> tuple[object, ...]:
|
|
535
|
+
return (
|
|
536
|
+
_reference_structure(source_map.generated),
|
|
537
|
+
tuple(
|
|
538
|
+
(
|
|
539
|
+
_span_structure(segment.generated),
|
|
540
|
+
_reference_structure(segment.origin_ref),
|
|
541
|
+
_span_structure(segment.origin),
|
|
542
|
+
segment.relation.value,
|
|
543
|
+
segment.synthetic_region,
|
|
544
|
+
_reference_structure(segment.anchor_ref),
|
|
545
|
+
_span_structure(segment.anchor_span),
|
|
546
|
+
)
|
|
547
|
+
for segment in source_map.segments
|
|
548
|
+
),
|
|
549
|
+
)
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def _mapped_source_structure(mapped: MappedSource) -> tuple[object, ...]:
|
|
553
|
+
if isinstance(mapped, CompactMappedWorkerSource) and isinstance(
|
|
554
|
+
mapped.source_map,
|
|
555
|
+
CompactReloadSourceMap,
|
|
556
|
+
):
|
|
557
|
+
local = mapped.local_source_map
|
|
558
|
+
assert isinstance(local, CompactReloadSourceMap)
|
|
559
|
+
return (
|
|
560
|
+
"compact-reload-v1",
|
|
561
|
+
id(mapped.text),
|
|
562
|
+
len(mapped.text),
|
|
563
|
+
_reference_structure(mapped.artifact),
|
|
564
|
+
mapped.source_map.source_map_sha256,
|
|
565
|
+
mapped.source_map._canonical_manifest_bytes,
|
|
566
|
+
local.source_map_sha256,
|
|
567
|
+
local._canonical_manifest_bytes,
|
|
568
|
+
)
|
|
569
|
+
try:
|
|
570
|
+
local_source_map = mapped.local_source_map
|
|
571
|
+
local_index = next(
|
|
572
|
+
(
|
|
573
|
+
index
|
|
574
|
+
for index, source_map in enumerate(mapped.lineage)
|
|
575
|
+
if source_map is local_source_map
|
|
576
|
+
),
|
|
577
|
+
None,
|
|
578
|
+
)
|
|
579
|
+
local_structure = _source_map_structure(local_source_map)
|
|
580
|
+
except ValueError:
|
|
581
|
+
local_index = None
|
|
582
|
+
local_structure = None
|
|
583
|
+
return (
|
|
584
|
+
id(mapped.text),
|
|
585
|
+
len(mapped.text),
|
|
586
|
+
_reference_structure(mapped.artifact),
|
|
587
|
+
_source_map_structure(mapped.source_map),
|
|
588
|
+
tuple(_source_map_structure(item) for item in mapped.lineage),
|
|
589
|
+
local_index,
|
|
590
|
+
local_structure,
|
|
591
|
+
)
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def _clone_source_reference(
|
|
595
|
+
reference: SourceUnitRef | SourceArtifactRef | None,
|
|
596
|
+
) -> SourceUnitRef | SourceArtifactRef | None:
|
|
597
|
+
if reference is None:
|
|
598
|
+
return None
|
|
599
|
+
return replace(reference)
|
|
600
|
+
|
|
601
|
+
|
|
602
|
+
def _clone_source_map(source_map: SourceMap) -> SourceMap:
|
|
603
|
+
return SourceMap(
|
|
604
|
+
replace(source_map.generated),
|
|
605
|
+
tuple(
|
|
606
|
+
SourceMapSegment(
|
|
607
|
+
generated=replace(segment.generated),
|
|
608
|
+
origin_ref=_clone_source_reference(segment.origin_ref),
|
|
609
|
+
origin=(
|
|
610
|
+
None if segment.origin is None else replace(segment.origin)
|
|
611
|
+
),
|
|
612
|
+
relation=segment.relation,
|
|
613
|
+
synthetic_region=segment.synthetic_region,
|
|
614
|
+
anchor_ref=_clone_source_reference(segment.anchor_ref),
|
|
615
|
+
anchor_span=(
|
|
616
|
+
None
|
|
617
|
+
if segment.anchor_span is None
|
|
618
|
+
else replace(segment.anchor_span)
|
|
619
|
+
),
|
|
620
|
+
)
|
|
621
|
+
for segment in source_map.segments
|
|
622
|
+
),
|
|
623
|
+
)
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
def _snapshot_mapped_source(
|
|
627
|
+
mapped: MappedSource,
|
|
628
|
+
) -> tuple[MappedSource, tuple[object, ...]]:
|
|
629
|
+
structure = _mapped_source_structure(mapped)
|
|
630
|
+
if isinstance(mapped, CompactMappedWorkerSource):
|
|
631
|
+
snapshot = snapshot_compact_mapped_worker_source(mapped)
|
|
632
|
+
if _mapped_source_structure(snapshot) != structure:
|
|
633
|
+
raise ProtocolError("Worker mapped source snapshot is invalid")
|
|
634
|
+
return snapshot, structure
|
|
635
|
+
map_cache: dict[int, SourceMap] = {}
|
|
636
|
+
|
|
637
|
+
def clone_map(source_map: SourceMap) -> SourceMap:
|
|
638
|
+
cloned = map_cache.get(id(source_map))
|
|
639
|
+
if cloned is None:
|
|
640
|
+
cloned = _clone_source_map(source_map)
|
|
641
|
+
map_cache[id(source_map)] = cloned
|
|
642
|
+
return cloned
|
|
643
|
+
|
|
644
|
+
source_map = clone_map(mapped.source_map)
|
|
645
|
+
lineage = tuple(clone_map(item) for item in mapped.lineage)
|
|
646
|
+
try:
|
|
647
|
+
local_source_map = clone_map(mapped.local_source_map)
|
|
648
|
+
except ValueError:
|
|
649
|
+
local_source_map = None
|
|
650
|
+
snapshot = object.__new__(MappedSource)
|
|
651
|
+
object.__setattr__(snapshot, "_text", mapped.text)
|
|
652
|
+
object.__setattr__(snapshot, "_artifact", replace(mapped.artifact))
|
|
653
|
+
object.__setattr__(snapshot, "_source_map", source_map)
|
|
654
|
+
object.__setattr__(snapshot, "_lineage", lineage)
|
|
655
|
+
object.__setattr__(snapshot, "_local_source_map", local_source_map)
|
|
656
|
+
if (
|
|
657
|
+
source_map.generated != snapshot.artifact
|
|
658
|
+
or any(
|
|
659
|
+
(
|
|
660
|
+
segment.relation.value != "synthetic"
|
|
661
|
+
and not isinstance(segment.origin_ref, SourceUnitRef)
|
|
662
|
+
)
|
|
663
|
+
or (
|
|
664
|
+
segment.anchor_ref is not None
|
|
665
|
+
and not isinstance(segment.anchor_ref, SourceUnitRef)
|
|
666
|
+
)
|
|
667
|
+
for segment in source_map.segments
|
|
668
|
+
)
|
|
669
|
+
or (
|
|
670
|
+
local_source_map is not None
|
|
671
|
+
and (
|
|
672
|
+
local_source_map.generated != snapshot.artifact
|
|
673
|
+
or not lineage
|
|
674
|
+
or lineage[-1] != local_source_map
|
|
675
|
+
)
|
|
676
|
+
)
|
|
677
|
+
or _mapped_source_structure(snapshot) != structure
|
|
678
|
+
):
|
|
679
|
+
raise ProtocolError("Worker mapped source snapshot is invalid")
|
|
680
|
+
return snapshot, structure
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
class _WorkerSourceCatalogValidation:
|
|
684
|
+
"""Exact semantic catalog result reusable only for unchanged source bytes."""
|
|
685
|
+
|
|
686
|
+
__slots__ = (
|
|
687
|
+
"__path",
|
|
688
|
+
"__source",
|
|
689
|
+
"__source_sha256",
|
|
690
|
+
"__catalog",
|
|
691
|
+
"__mapped_source_snapshot",
|
|
692
|
+
"__mapped_source_structure",
|
|
693
|
+
"__path_read_confirmed",
|
|
694
|
+
"__proof",
|
|
695
|
+
)
|
|
696
|
+
|
|
697
|
+
def __init__(
|
|
698
|
+
self,
|
|
699
|
+
path: Path,
|
|
700
|
+
source: bytes,
|
|
701
|
+
source_sha256: str,
|
|
702
|
+
catalog: tuple[WorkerExport, ...],
|
|
703
|
+
mapped_source_snapshot: MappedSource | None,
|
|
704
|
+
mapped_source_structure: tuple[object, ...] | None,
|
|
705
|
+
path_read_confirmed: bool,
|
|
706
|
+
proof: bytes,
|
|
707
|
+
) -> None:
|
|
708
|
+
object.__setattr__(self, "_WorkerSourceCatalogValidation__path", path)
|
|
709
|
+
object.__setattr__(self, "_WorkerSourceCatalogValidation__source", source)
|
|
710
|
+
object.__setattr__(
|
|
711
|
+
self,
|
|
712
|
+
"_WorkerSourceCatalogValidation__source_sha256",
|
|
713
|
+
source_sha256,
|
|
714
|
+
)
|
|
715
|
+
object.__setattr__(self, "_WorkerSourceCatalogValidation__catalog", catalog)
|
|
716
|
+
object.__setattr__(
|
|
717
|
+
self,
|
|
718
|
+
"_WorkerSourceCatalogValidation__mapped_source_snapshot",
|
|
719
|
+
mapped_source_snapshot,
|
|
720
|
+
)
|
|
721
|
+
object.__setattr__(
|
|
722
|
+
self,
|
|
723
|
+
"_WorkerSourceCatalogValidation__mapped_source_structure",
|
|
724
|
+
mapped_source_structure,
|
|
725
|
+
)
|
|
726
|
+
object.__setattr__(
|
|
727
|
+
self,
|
|
728
|
+
"_WorkerSourceCatalogValidation__path_read_confirmed",
|
|
729
|
+
path_read_confirmed,
|
|
730
|
+
)
|
|
731
|
+
object.__setattr__(self, "_WorkerSourceCatalogValidation__proof", proof)
|
|
732
|
+
|
|
733
|
+
def __setattr__(self, name: str, value: object) -> None:
|
|
734
|
+
raise AttributeError("Worker source catalog validations are immutable")
|
|
735
|
+
|
|
736
|
+
def __repr__(self) -> str:
|
|
737
|
+
return "<redacted worker source catalog validation>"
|
|
738
|
+
|
|
739
|
+
def contents(
|
|
740
|
+
self,
|
|
741
|
+
) -> tuple[
|
|
742
|
+
Path,
|
|
743
|
+
bytes,
|
|
744
|
+
str,
|
|
745
|
+
tuple[WorkerExport, ...],
|
|
746
|
+
MappedSource | None,
|
|
747
|
+
tuple[object, ...] | None,
|
|
748
|
+
bool,
|
|
749
|
+
bytes,
|
|
750
|
+
]:
|
|
751
|
+
return (
|
|
752
|
+
self.__path,
|
|
753
|
+
self.__source,
|
|
754
|
+
self.__source_sha256,
|
|
755
|
+
self.__catalog,
|
|
756
|
+
self.__mapped_source_snapshot,
|
|
757
|
+
self.__mapped_source_structure,
|
|
758
|
+
self.__path_read_confirmed,
|
|
759
|
+
self.__proof,
|
|
760
|
+
)
|
|
761
|
+
|
|
762
|
+
|
|
763
|
+
def _source_catalog_proof(
|
|
764
|
+
path: Path,
|
|
765
|
+
source: bytes,
|
|
766
|
+
source_sha256: str,
|
|
767
|
+
catalog: tuple[WorkerExport, ...],
|
|
768
|
+
mapped_source_snapshot: MappedSource | None = None,
|
|
769
|
+
mapped_source_structure: tuple[object, ...] | None = None,
|
|
770
|
+
*,
|
|
771
|
+
path_read_confirmed: bool = False,
|
|
772
|
+
) -> bytes:
|
|
773
|
+
fields = (
|
|
774
|
+
str(path),
|
|
775
|
+
source_sha256,
|
|
776
|
+
str(id(source)),
|
|
777
|
+
str(id(mapped_source_snapshot)),
|
|
778
|
+
str(id(mapped_source_structure)),
|
|
779
|
+
"1" if path_read_confirmed else "0",
|
|
780
|
+
*(
|
|
781
|
+
f"{item.public_path}\0{item.method}\0{item.receiver_module or ''}"
|
|
782
|
+
for item in catalog
|
|
783
|
+
),
|
|
784
|
+
)
|
|
785
|
+
return digest(
|
|
786
|
+
_SOURCE_CATALOG_PROOF_KEY,
|
|
787
|
+
"\x1f".join(fields).encode("utf-8"),
|
|
788
|
+
"sha256",
|
|
789
|
+
)
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
def validate_worker_source_catalog(
|
|
793
|
+
source_path: Path,
|
|
794
|
+
exports: tuple[WorkerExport, ...],
|
|
795
|
+
) -> _WorkerSourceCatalogValidation:
|
|
796
|
+
"""Semantically validate one exact source/catalog pair for later packaging."""
|
|
797
|
+
resolved = Path(source_path).resolve()
|
|
798
|
+
if not resolved.is_file():
|
|
799
|
+
raise ProtocolError("Worker source file does not exist")
|
|
800
|
+
source = resolved.read_bytes()
|
|
801
|
+
catalog = _validate_source_catalog(resolved, source, exports)
|
|
802
|
+
source_hash = sha256(source).hexdigest()
|
|
803
|
+
return _WorkerSourceCatalogValidation(
|
|
804
|
+
resolved,
|
|
805
|
+
source,
|
|
806
|
+
source_hash,
|
|
807
|
+
catalog,
|
|
808
|
+
None,
|
|
809
|
+
None,
|
|
810
|
+
False,
|
|
811
|
+
_source_catalog_proof(resolved, source, source_hash, catalog),
|
|
812
|
+
)
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
def _validated_source_catalog_snapshot(
|
|
816
|
+
validation: _WorkerSourceCatalogValidation,
|
|
817
|
+
) -> tuple[
|
|
818
|
+
Path,
|
|
819
|
+
bytes,
|
|
820
|
+
str,
|
|
821
|
+
tuple[WorkerExport, ...],
|
|
822
|
+
MappedSource | None,
|
|
823
|
+
tuple[object, ...] | None,
|
|
824
|
+
bool,
|
|
825
|
+
]:
|
|
826
|
+
if not isinstance(validation, _WorkerSourceCatalogValidation):
|
|
827
|
+
raise ProtocolError("Worker source catalog validation is invalid")
|
|
828
|
+
try:
|
|
829
|
+
(
|
|
830
|
+
path,
|
|
831
|
+
source,
|
|
832
|
+
source_hash,
|
|
833
|
+
catalog,
|
|
834
|
+
mapped_snapshot,
|
|
835
|
+
mapped_structure,
|
|
836
|
+
path_read_confirmed,
|
|
837
|
+
proof,
|
|
838
|
+
) = validation.contents()
|
|
839
|
+
valid = compare_digest(
|
|
840
|
+
proof,
|
|
841
|
+
_source_catalog_proof(
|
|
842
|
+
path,
|
|
843
|
+
source,
|
|
844
|
+
source_hash,
|
|
845
|
+
catalog,
|
|
846
|
+
mapped_snapshot,
|
|
847
|
+
mapped_structure,
|
|
848
|
+
path_read_confirmed=path_read_confirmed,
|
|
849
|
+
),
|
|
850
|
+
)
|
|
851
|
+
except Exception:
|
|
852
|
+
raise ProtocolError("Worker source catalog validation is invalid") from None
|
|
853
|
+
if not valid:
|
|
854
|
+
raise ProtocolError("Worker source catalog validation is invalid")
|
|
855
|
+
return (
|
|
856
|
+
path,
|
|
857
|
+
source,
|
|
858
|
+
source_hash,
|
|
859
|
+
catalog,
|
|
860
|
+
mapped_snapshot,
|
|
861
|
+
mapped_structure,
|
|
862
|
+
path_read_confirmed,
|
|
863
|
+
)
|
|
864
|
+
|
|
865
|
+
|
|
866
|
+
def reuse_worker_source_catalog_validation(
|
|
867
|
+
validation: _WorkerSourceCatalogValidation,
|
|
868
|
+
source_path: Path,
|
|
869
|
+
exports: tuple[WorkerExport, ...],
|
|
870
|
+
) -> _WorkerSourceCatalogValidation:
|
|
871
|
+
"""Bind a proven catalog to another path only for byte-identical source."""
|
|
872
|
+
_, validated_source, _, validated_catalog, _, _, _ = (
|
|
873
|
+
_validated_source_catalog_snapshot(validation)
|
|
874
|
+
)
|
|
875
|
+
resolved = Path(source_path).resolve()
|
|
876
|
+
catalog = validate_worker_export_catalog(exports, require_nonempty=True)
|
|
877
|
+
try:
|
|
878
|
+
source = resolved.read_bytes()
|
|
879
|
+
except OSError:
|
|
880
|
+
raise ProtocolError("Worker source catalog validation is invalid") from None
|
|
881
|
+
if source != validated_source or catalog != validated_catalog:
|
|
882
|
+
raise ProtocolError("Worker source catalog validation is invalid")
|
|
883
|
+
source_hash = sha256(source).hexdigest()
|
|
884
|
+
return _WorkerSourceCatalogValidation(
|
|
885
|
+
resolved,
|
|
886
|
+
source,
|
|
887
|
+
source_hash,
|
|
888
|
+
catalog,
|
|
889
|
+
None,
|
|
890
|
+
None,
|
|
891
|
+
False,
|
|
892
|
+
_source_catalog_proof(resolved, source, source_hash, catalog),
|
|
893
|
+
)
|
|
894
|
+
|
|
895
|
+
|
|
896
|
+
def _reuse_worker_source_catalog(
|
|
897
|
+
validation: _WorkerSourceCatalogValidation,
|
|
898
|
+
source_path: Path,
|
|
899
|
+
exports: tuple[WorkerExport, ...],
|
|
900
|
+
mapped_source: MappedSource | None,
|
|
901
|
+
) -> tuple[tuple[WorkerExport, ...], bytes, str, MappedSource | None]:
|
|
902
|
+
semantic_snapshot = False
|
|
903
|
+
try:
|
|
904
|
+
(
|
|
905
|
+
validated_path,
|
|
906
|
+
validated_source,
|
|
907
|
+
source_hash,
|
|
908
|
+
catalog,
|
|
909
|
+
mapped_snapshot,
|
|
910
|
+
mapped_structure,
|
|
911
|
+
path_read_confirmed,
|
|
912
|
+
) = _validated_source_catalog_snapshot(validation)
|
|
913
|
+
semantic_snapshot = mapped_structure is not None
|
|
914
|
+
source = (
|
|
915
|
+
validated_source
|
|
916
|
+
if path_read_confirmed
|
|
917
|
+
else source_path.read_bytes()
|
|
918
|
+
)
|
|
919
|
+
valid = (
|
|
920
|
+
validated_path == source_path
|
|
921
|
+
and validated_source == source
|
|
922
|
+
and catalog == exports
|
|
923
|
+
and (
|
|
924
|
+
mapped_structure is None
|
|
925
|
+
or (
|
|
926
|
+
mapped_source is not None
|
|
927
|
+
and _mapped_source_structure(mapped_source)
|
|
928
|
+
== mapped_structure
|
|
929
|
+
)
|
|
930
|
+
)
|
|
931
|
+
)
|
|
932
|
+
except Exception:
|
|
933
|
+
message = (
|
|
934
|
+
"Worker module artifact admission is invalid"
|
|
935
|
+
if semantic_snapshot
|
|
936
|
+
else "Worker source catalog validation is invalid"
|
|
937
|
+
)
|
|
938
|
+
raise ProtocolError(message) from None
|
|
939
|
+
if not valid:
|
|
940
|
+
message = (
|
|
941
|
+
"Worker module artifact admission is invalid"
|
|
942
|
+
if semantic_snapshot
|
|
943
|
+
else "Worker source catalog validation is invalid"
|
|
944
|
+
)
|
|
945
|
+
raise ProtocolError(message)
|
|
946
|
+
return catalog, source, source_hash, mapped_snapshot
|
|
947
|
+
|
|
948
|
+
|
|
949
|
+
def _semantic_worker_source_catalog_validation(
|
|
950
|
+
admission: WorkerSemanticAdmission,
|
|
951
|
+
source: MappedSource,
|
|
952
|
+
source_path: Path,
|
|
953
|
+
source_bytes: bytes,
|
|
954
|
+
exports: tuple[WorkerExport, ...],
|
|
955
|
+
*,
|
|
956
|
+
expected_packer_identity: str,
|
|
957
|
+
) -> _WorkerSourceCatalogValidation:
|
|
958
|
+
"""Bind a semantic result to the one generated-file read-back."""
|
|
959
|
+
try:
|
|
960
|
+
catalog = validate_worker_export_catalog(exports, require_nonempty=True)
|
|
961
|
+
lowered = _validate_worker_semantic_admission(
|
|
962
|
+
admission,
|
|
963
|
+
source=source,
|
|
964
|
+
exported_methods=tuple(item.method for item in catalog),
|
|
965
|
+
expected_packer_identity=expected_packer_identity,
|
|
966
|
+
)
|
|
967
|
+
logical_name = lowered.analysis.unit.logical_name
|
|
968
|
+
expected_identity = tuple(
|
|
969
|
+
sorted(
|
|
970
|
+
(
|
|
971
|
+
f"{logical_name}.{method}".casefold(),
|
|
972
|
+
method.casefold(),
|
|
973
|
+
logical_name.casefold(),
|
|
974
|
+
)
|
|
975
|
+
for method in lowered.analysis.exported_methods
|
|
976
|
+
)
|
|
977
|
+
)
|
|
978
|
+
if (
|
|
979
|
+
source_bytes != source.text.encode("utf-8")
|
|
980
|
+
or _worker_export_catalog_identity(catalog) != expected_identity
|
|
981
|
+
):
|
|
982
|
+
raise ValueError
|
|
983
|
+
resolved = source_path.resolve()
|
|
984
|
+
source_hash = sha256(source_bytes).hexdigest()
|
|
985
|
+
if source_hash != source.artifact.source_sha256:
|
|
986
|
+
raise ValueError
|
|
987
|
+
mapped_snapshot, mapped_structure = _snapshot_mapped_source(source)
|
|
988
|
+
return _WorkerSourceCatalogValidation(
|
|
989
|
+
resolved,
|
|
990
|
+
source_bytes,
|
|
991
|
+
source_hash,
|
|
992
|
+
catalog,
|
|
993
|
+
mapped_snapshot,
|
|
994
|
+
mapped_structure,
|
|
995
|
+
True,
|
|
996
|
+
_source_catalog_proof(
|
|
997
|
+
resolved,
|
|
998
|
+
source_bytes,
|
|
999
|
+
source_hash,
|
|
1000
|
+
catalog,
|
|
1001
|
+
mapped_snapshot,
|
|
1002
|
+
mapped_structure,
|
|
1003
|
+
path_read_confirmed=True,
|
|
1004
|
+
),
|
|
1005
|
+
)
|
|
1006
|
+
except Exception:
|
|
1007
|
+
raise ProtocolError("Worker module artifact admission is invalid") from None
|
|
1008
|
+
|
|
1009
|
+
|
|
1010
|
+
def validate_worker_export_catalog(
|
|
1011
|
+
exports: tuple[WorkerExport, ...],
|
|
1012
|
+
*,
|
|
1013
|
+
require_nonempty: bool = False,
|
|
1014
|
+
) -> tuple[WorkerExport, ...]:
|
|
1015
|
+
"""Validate a concrete public worker catalog before any runtime instruction."""
|
|
1016
|
+
catalog: dict[str, WorkerExport] = {}
|
|
1017
|
+
for export in exports:
|
|
1018
|
+
if not isinstance(export, WorkerExport):
|
|
1019
|
+
raise ProtocolError("Worker export catalog is invalid")
|
|
1020
|
+
key = export.public_path.casefold()
|
|
1021
|
+
if key in catalog:
|
|
1022
|
+
raise ValueError(f"duplicate worker export {export.public_path!r}")
|
|
1023
|
+
catalog[key] = export
|
|
1024
|
+
if require_nonempty and not catalog:
|
|
1025
|
+
raise ProtocolError("Worker artifact must declare a nonempty export catalog")
|
|
1026
|
+
return tuple(catalog.values())
|
|
1027
|
+
|
|
1028
|
+
|
|
1029
|
+
def build_worker_artifact(
|
|
1030
|
+
*,
|
|
1031
|
+
logical_name: str,
|
|
1032
|
+
source_path: Path,
|
|
1033
|
+
artifact_path: Path,
|
|
1034
|
+
expected_version: str | None = None,
|
|
1035
|
+
expected_value: int | None = None,
|
|
1036
|
+
exports: tuple[WorkerExport, ...],
|
|
1037
|
+
source_validation: _WorkerSourceCatalogValidation | None = None,
|
|
1038
|
+
) -> WorkerArtifact:
|
|
1039
|
+
"""Compatibility builder for source-only Worker packages.
|
|
1040
|
+
|
|
1041
|
+
Notebook production uses ``NotebookWorkerArtifactBuilder`` with mapped input;
|
|
1042
|
+
source-only callers receive an explicitly synthetic diagnostic origin.
|
|
1043
|
+
"""
|
|
1044
|
+
if (expected_version is None) != (expected_value is None):
|
|
1045
|
+
raise ValueError(
|
|
1046
|
+
"Worker artifact expected_version and expected_value must be both set or both omitted"
|
|
1047
|
+
)
|
|
1048
|
+
source_path = source_path.resolve()
|
|
1049
|
+
artifact_path = artifact_path.resolve()
|
|
1050
|
+
catalog, source_bytes, artifact_bytes, source_hash, _ = _validate_artifact_files(
|
|
1051
|
+
source_path,
|
|
1052
|
+
artifact_path,
|
|
1053
|
+
exports,
|
|
1054
|
+
source_validation=source_validation,
|
|
1055
|
+
)
|
|
1056
|
+
from onec_runtime.worker_epf import _synthetic_worker_module_source
|
|
1057
|
+
|
|
1058
|
+
mapped = _synthetic_worker_module_source(
|
|
1059
|
+
source_bytes.decode("utf-8"),
|
|
1060
|
+
normalize=False,
|
|
1061
|
+
)
|
|
1062
|
+
return _build_admitted_worker_artifact(
|
|
1063
|
+
logical_name=logical_name,
|
|
1064
|
+
source_path=source_path,
|
|
1065
|
+
artifact_path=artifact_path,
|
|
1066
|
+
source_bytes=source_bytes,
|
|
1067
|
+
artifact_bytes=artifact_bytes,
|
|
1068
|
+
source_bytes_sha256=source_hash,
|
|
1069
|
+
mapped=mapped,
|
|
1070
|
+
visible_context_snapshot=None,
|
|
1071
|
+
expected_version=expected_version,
|
|
1072
|
+
expected_value=expected_value,
|
|
1073
|
+
exports=catalog,
|
|
1074
|
+
)
|
|
1075
|
+
|
|
1076
|
+
|
|
1077
|
+
def _build_admitted_worker_artifact(
|
|
1078
|
+
*,
|
|
1079
|
+
logical_name: str,
|
|
1080
|
+
source_path: Path,
|
|
1081
|
+
artifact_path: Path,
|
|
1082
|
+
source_bytes: bytes,
|
|
1083
|
+
artifact_bytes: bytes,
|
|
1084
|
+
source_bytes_sha256: str,
|
|
1085
|
+
mapped: MappedSource,
|
|
1086
|
+
visible_context_snapshot: _WorkerVisibleContextSnapshot | None,
|
|
1087
|
+
expected_version: str | None,
|
|
1088
|
+
expected_value: int | None,
|
|
1089
|
+
exports: tuple[WorkerExport, ...],
|
|
1090
|
+
binary_snapshot: _AdmittedWorkerBinary | None = None,
|
|
1091
|
+
mapped_snapshot: bool = False,
|
|
1092
|
+
) -> WorkerArtifact:
|
|
1093
|
+
if not mapped_snapshot:
|
|
1094
|
+
mapped, _ = _snapshot_mapped_source(mapped)
|
|
1095
|
+
if mapped.artifact.kind is not SourceArtifactKind.WORKER_MODULE:
|
|
1096
|
+
raise ProtocolError("Worker admission requires a final WORKER_MODULE map")
|
|
1097
|
+
if binary_snapshot is None:
|
|
1098
|
+
if (
|
|
1099
|
+
mapped.text.encode("utf-8") != source_bytes
|
|
1100
|
+
or fullmatch(r"[0-9a-f]{64}", source_bytes_sha256) is None
|
|
1101
|
+
):
|
|
1102
|
+
raise ProtocolError("Worker mapped source does not match source snapshot")
|
|
1103
|
+
binary = _AdmittedWorkerBinary(
|
|
1104
|
+
source_bytes,
|
|
1105
|
+
artifact_bytes,
|
|
1106
|
+
source_bytes_sha256,
|
|
1107
|
+
sha256(artifact_bytes).hexdigest(),
|
|
1108
|
+
)
|
|
1109
|
+
else:
|
|
1110
|
+
binary = binary_snapshot
|
|
1111
|
+
if (
|
|
1112
|
+
source_bytes is not binary.source_bytes
|
|
1113
|
+
or artifact_bytes is not binary.artifact_bytes
|
|
1114
|
+
or source_bytes_sha256 != binary.source_sha256
|
|
1115
|
+
):
|
|
1116
|
+
raise ProtocolError("Worker mapped source does not match source snapshot")
|
|
1117
|
+
if binary.source_sha256 != mapped.artifact.source_sha256:
|
|
1118
|
+
raise ProtocolError("Worker mapped source does not match source snapshot")
|
|
1119
|
+
source_map_hash = mapped.source_map_sha256
|
|
1120
|
+
provenance = WorkerSourceProvenance(
|
|
1121
|
+
mapped.artifact.source_sha256,
|
|
1122
|
+
source_map_hash,
|
|
1123
|
+
mapped.artifact.worker_generation,
|
|
1124
|
+
mapped.artifact.worker_manifest_sha256,
|
|
1125
|
+
)
|
|
1126
|
+
artifact = WorkerArtifact(
|
|
1127
|
+
logical_name=logical_name,
|
|
1128
|
+
source_sha256=mapped.artifact.source_sha256,
|
|
1129
|
+
artifact_sha256=binary.artifact_sha256,
|
|
1130
|
+
exports=exports,
|
|
1131
|
+
source_provenance=provenance,
|
|
1132
|
+
)
|
|
1133
|
+
context_fingerprint = (
|
|
1134
|
+
"" if visible_context_snapshot is None else visible_context_snapshot.fingerprint
|
|
1135
|
+
)
|
|
1136
|
+
snapshot = _AdmittedWorkerSnapshot(
|
|
1137
|
+
binary=binary,
|
|
1138
|
+
proof=b"",
|
|
1139
|
+
mapped_source=mapped,
|
|
1140
|
+
mapped_guard=_mapped_source_guard(mapped),
|
|
1141
|
+
source_map_sha256=source_map_hash,
|
|
1142
|
+
worker_generation=mapped.artifact.worker_generation,
|
|
1143
|
+
worker_manifest_sha256=mapped.artifact.worker_manifest_sha256,
|
|
1144
|
+
catalog=exports,
|
|
1145
|
+
visible_context_snapshot=visible_context_snapshot,
|
|
1146
|
+
visible_context_fingerprint=context_fingerprint,
|
|
1147
|
+
visible_context_guard=_visible_context_snapshot_guard(
|
|
1148
|
+
visible_context_snapshot
|
|
1149
|
+
),
|
|
1150
|
+
source_path=source_path,
|
|
1151
|
+
artifact_path=artifact_path,
|
|
1152
|
+
expected_version=expected_version,
|
|
1153
|
+
expected_value=expected_value,
|
|
1154
|
+
)
|
|
1155
|
+
snapshot = replace(snapshot, proof=_artifact_proof(artifact, snapshot))
|
|
1156
|
+
admission = _WorkerArtifactAdmission(artifact, snapshot)
|
|
1157
|
+
_bind_worker_artifact_capability(
|
|
1158
|
+
artifact,
|
|
1159
|
+
artifact_path=artifact_path,
|
|
1160
|
+
source_path=source_path,
|
|
1161
|
+
expected_version=expected_version,
|
|
1162
|
+
expected_value=expected_value,
|
|
1163
|
+
admission=admission,
|
|
1164
|
+
)
|
|
1165
|
+
return artifact
|
|
1166
|
+
|
|
1167
|
+
|
|
1168
|
+
class NotebookWorkerArtifactBuilder:
|
|
1169
|
+
"""Build a generic worker EPF from generated-AST method source.
|
|
1170
|
+
|
|
1171
|
+
The builder writes only runtime-generated worker inputs. Candidate admission
|
|
1172
|
+
and the eventual module-generation publication remain owned by the
|
|
1173
|
+
Worker universe registry.
|
|
1174
|
+
"""
|
|
1175
|
+
|
|
1176
|
+
def __init__(self, config: RuntimeConfig, *, logical_name: str = "Worker") -> None:
|
|
1177
|
+
self._config = config
|
|
1178
|
+
self._logical_name = logical_name
|
|
1179
|
+
|
|
1180
|
+
def __call__(
|
|
1181
|
+
self,
|
|
1182
|
+
source: MappedSource,
|
|
1183
|
+
exports: tuple[WorkerExport, ...],
|
|
1184
|
+
*,
|
|
1185
|
+
visible_source_context: VisibleSourceContext,
|
|
1186
|
+
semantic_admission: WorkerSemanticAdmission | None = None,
|
|
1187
|
+
semantic_packer_identity: str | None = None,
|
|
1188
|
+
) -> WorkerArtifact:
|
|
1189
|
+
if not isinstance(source, MappedSource):
|
|
1190
|
+
raise TypeError("Notebook Worker source must be a MappedSource")
|
|
1191
|
+
if not source.text.strip():
|
|
1192
|
+
raise ProtocolError("Notebook worker source is empty")
|
|
1193
|
+
catalog = validate_worker_export_catalog(exports, require_nonempty=True)
|
|
1194
|
+
from onec_runtime.worker_epf import (
|
|
1195
|
+
build_worker_epf,
|
|
1196
|
+
prepare_worker_module_source,
|
|
1197
|
+
)
|
|
1198
|
+
|
|
1199
|
+
mapped = (
|
|
1200
|
+
source
|
|
1201
|
+
if (
|
|
1202
|
+
semantic_admission is not None
|
|
1203
|
+
and source.artifact.kind is SourceArtifactKind.WORKER_MODULE
|
|
1204
|
+
)
|
|
1205
|
+
else prepare_worker_module_source(source)
|
|
1206
|
+
)
|
|
1207
|
+
visible_context_snapshot = _snapshot_visible_source_context(
|
|
1208
|
+
mapped,
|
|
1209
|
+
visible_source_context,
|
|
1210
|
+
required=True,
|
|
1211
|
+
)
|
|
1212
|
+
source_hash = mapped.artifact.source_sha256
|
|
1213
|
+
runtime_dir = self._config.runtime_dir
|
|
1214
|
+
build_dir = self._config.build_dir
|
|
1215
|
+
root = runtime_dir / "generated" / "notebook-workers" / source_hash
|
|
1216
|
+
source_module = root / "Worker" / "Ext" / "ObjectModule.bsl"
|
|
1217
|
+
artifact_path = build_dir / "notebook-workers" / f"{source_hash}.epf"
|
|
1218
|
+
source_module.parent.mkdir(parents=True, exist_ok=True)
|
|
1219
|
+
source_module.write_text(mapped.text, encoding="utf-8", newline="\n")
|
|
1220
|
+
source_validation = None
|
|
1221
|
+
if semantic_admission is not None:
|
|
1222
|
+
if semantic_packer_identity is None:
|
|
1223
|
+
raise ProtocolError("Worker module artifact admission is invalid")
|
|
1224
|
+
try:
|
|
1225
|
+
source_bytes = source_module.read_bytes()
|
|
1226
|
+
except OSError:
|
|
1227
|
+
raise ProtocolError(
|
|
1228
|
+
"Worker module artifact admission is invalid"
|
|
1229
|
+
) from None
|
|
1230
|
+
source_validation = _semantic_worker_source_catalog_validation(
|
|
1231
|
+
semantic_admission,
|
|
1232
|
+
source,
|
|
1233
|
+
source_module,
|
|
1234
|
+
source_bytes,
|
|
1235
|
+
catalog,
|
|
1236
|
+
expected_packer_identity=semantic_packer_identity,
|
|
1237
|
+
)
|
|
1238
|
+
elif semantic_packer_identity is not None:
|
|
1239
|
+
raise ProtocolError("Worker module artifact admission is invalid")
|
|
1240
|
+
build_worker_epf(mapped, artifact_path)
|
|
1241
|
+
(
|
|
1242
|
+
catalog,
|
|
1243
|
+
source_bytes,
|
|
1244
|
+
artifact_bytes,
|
|
1245
|
+
source_hash,
|
|
1246
|
+
admitted_mapped,
|
|
1247
|
+
) = _validate_artifact_files(
|
|
1248
|
+
source_module,
|
|
1249
|
+
artifact_path,
|
|
1250
|
+
catalog,
|
|
1251
|
+
source_validation=source_validation,
|
|
1252
|
+
mapped_source=mapped,
|
|
1253
|
+
)
|
|
1254
|
+
return _build_admitted_worker_artifact(
|
|
1255
|
+
logical_name=self._logical_name,
|
|
1256
|
+
source_path=source_module,
|
|
1257
|
+
artifact_path=artifact_path,
|
|
1258
|
+
source_bytes=source_bytes,
|
|
1259
|
+
artifact_bytes=artifact_bytes,
|
|
1260
|
+
source_bytes_sha256=source_hash,
|
|
1261
|
+
mapped=mapped if admitted_mapped is None else admitted_mapped,
|
|
1262
|
+
visible_context_snapshot=visible_context_snapshot,
|
|
1263
|
+
expected_version=None,
|
|
1264
|
+
expected_value=None,
|
|
1265
|
+
exports=catalog,
|
|
1266
|
+
mapped_snapshot=admitted_mapped is not None,
|
|
1267
|
+
)
|
|
1268
|
+
|
|
1269
|
+
|
|
1270
|
+
def rebind_worker_artifact_binary(
|
|
1271
|
+
artifact: WorkerArtifact,
|
|
1272
|
+
*,
|
|
1273
|
+
logical_name: str,
|
|
1274
|
+
mapped: MappedSource,
|
|
1275
|
+
visible_source_context: VisibleSourceContext,
|
|
1276
|
+
exports: tuple[WorkerExport, ...],
|
|
1277
|
+
) -> WorkerArtifact:
|
|
1278
|
+
"""Bind admitted byte-identical EPF content to one exact source-map revision."""
|
|
1279
|
+
try:
|
|
1280
|
+
admitted_catalog = validate_production_worker_artifact(artifact)
|
|
1281
|
+
admitted = _validated_admitted_snapshot(artifact)
|
|
1282
|
+
admitted_mapped = admitted.mapped_source
|
|
1283
|
+
catalog = validate_worker_export_catalog(exports, require_nonempty=True)
|
|
1284
|
+
if (
|
|
1285
|
+
not isinstance(mapped, MappedSource)
|
|
1286
|
+
or mapped.artifact.kind is not SourceArtifactKind.WORKER_MODULE
|
|
1287
|
+
or mapped.text != admitted_mapped.text
|
|
1288
|
+
or mapped.artifact.source_sha256 != admitted_mapped.artifact.source_sha256
|
|
1289
|
+
or _worker_export_catalog_identity(catalog)
|
|
1290
|
+
!= _worker_export_catalog_identity(admitted_catalog)
|
|
1291
|
+
):
|
|
1292
|
+
raise ProtocolError("Worker binary rebind admission is invalid")
|
|
1293
|
+
snapshot = _snapshot_visible_source_context(
|
|
1294
|
+
mapped,
|
|
1295
|
+
visible_source_context,
|
|
1296
|
+
required=True,
|
|
1297
|
+
)
|
|
1298
|
+
return _build_admitted_worker_artifact(
|
|
1299
|
+
logical_name=logical_name,
|
|
1300
|
+
source_path=admitted.source_path,
|
|
1301
|
+
artifact_path=admitted.artifact_path,
|
|
1302
|
+
source_bytes=admitted.source_bytes,
|
|
1303
|
+
artifact_bytes=admitted.artifact_bytes,
|
|
1304
|
+
source_bytes_sha256=admitted.source_sha256,
|
|
1305
|
+
mapped=mapped,
|
|
1306
|
+
visible_context_snapshot=snapshot,
|
|
1307
|
+
expected_version=admitted.expected_version,
|
|
1308
|
+
expected_value=admitted.expected_value,
|
|
1309
|
+
exports=catalog,
|
|
1310
|
+
binary_snapshot=admitted.binary,
|
|
1311
|
+
)
|
|
1312
|
+
except ProtocolError:
|
|
1313
|
+
raise
|
|
1314
|
+
except Exception:
|
|
1315
|
+
raise ProtocolError("Worker binary rebind admission is invalid") from None
|
|
1316
|
+
|
|
1317
|
+
|
|
1318
|
+
def _worker_export_catalog_identity(
|
|
1319
|
+
catalog: tuple[WorkerExport, ...],
|
|
1320
|
+
) -> tuple[tuple[str, str, str | None], ...]:
|
|
1321
|
+
return tuple(
|
|
1322
|
+
sorted(
|
|
1323
|
+
(
|
|
1324
|
+
item.public_path.casefold(),
|
|
1325
|
+
item.method.casefold(),
|
|
1326
|
+
(
|
|
1327
|
+
None
|
|
1328
|
+
if item.receiver_module is None
|
|
1329
|
+
else item.receiver_module.casefold()
|
|
1330
|
+
),
|
|
1331
|
+
)
|
|
1332
|
+
for item in catalog
|
|
1333
|
+
)
|
|
1334
|
+
)
|
|
1335
|
+
|
|
1336
|
+
|
|
1337
|
+
def _mapped_source_guard(mapped: MappedSource) -> tuple[int, ...]:
|
|
1338
|
+
"""Cheap identity fence for objects that are immutable after admission."""
|
|
1339
|
+
source_map = mapped.source_map
|
|
1340
|
+
lineage = mapped.lineage
|
|
1341
|
+
if isinstance(source_map, CompactReloadSourceMap):
|
|
1342
|
+
local = mapped.local_source_map
|
|
1343
|
+
if not isinstance(local, CompactReloadSourceMap):
|
|
1344
|
+
raise ProtocolError("Worker mapped source snapshot is invalid")
|
|
1345
|
+
return (
|
|
1346
|
+
id(mapped),
|
|
1347
|
+
id(mapped.text),
|
|
1348
|
+
id(mapped.artifact),
|
|
1349
|
+
*source_map.compact_guard(),
|
|
1350
|
+
id(lineage),
|
|
1351
|
+
*local.compact_guard(),
|
|
1352
|
+
)
|
|
1353
|
+
guard = [
|
|
1354
|
+
id(mapped),
|
|
1355
|
+
id(mapped.text),
|
|
1356
|
+
id(mapped.artifact),
|
|
1357
|
+
id(source_map),
|
|
1358
|
+
id(source_map.segments),
|
|
1359
|
+
id(lineage),
|
|
1360
|
+
]
|
|
1361
|
+
for item in lineage:
|
|
1362
|
+
guard.extend((id(item), id(item.segments)))
|
|
1363
|
+
return tuple(guard)
|
|
1364
|
+
|
|
1365
|
+
|
|
1366
|
+
def _visible_context_snapshot_guard(
|
|
1367
|
+
snapshot: _WorkerVisibleContextSnapshot | None,
|
|
1368
|
+
) -> tuple[object, ...]:
|
|
1369
|
+
if snapshot is None:
|
|
1370
|
+
return ()
|
|
1371
|
+
try:
|
|
1372
|
+
context = snapshot.context
|
|
1373
|
+
indices = context._indices # type: ignore[attr-defined]
|
|
1374
|
+
entries = tuple(
|
|
1375
|
+
sorted(
|
|
1376
|
+
(
|
|
1377
|
+
getattr(kind, "value", str(kind)),
|
|
1378
|
+
unit_id,
|
|
1379
|
+
revision,
|
|
1380
|
+
source_hash,
|
|
1381
|
+
id(entry),
|
|
1382
|
+
id(line_index),
|
|
1383
|
+
line_index._length, # type: ignore[attr-defined]
|
|
1384
|
+
id(line_index._line_starts), # type: ignore[attr-defined]
|
|
1385
|
+
)
|
|
1386
|
+
for (kind, unit_id, revision), entry in indices.items()
|
|
1387
|
+
for source_hash, line_index in (entry,)
|
|
1388
|
+
)
|
|
1389
|
+
)
|
|
1390
|
+
except Exception:
|
|
1391
|
+
raise ProtocolError(
|
|
1392
|
+
"Worker visible source context fingerprint does not match"
|
|
1393
|
+
) from None
|
|
1394
|
+
return (id(snapshot), id(context), id(indices), entries)
|
|
1395
|
+
|
|
1396
|
+
|
|
1397
|
+
def _snapshot_visible_source_context(
|
|
1398
|
+
mapped: MappedSource,
|
|
1399
|
+
visible_source_context: VisibleSourceContext | None,
|
|
1400
|
+
*,
|
|
1401
|
+
required: bool,
|
|
1402
|
+
) -> _WorkerVisibleContextSnapshot | None:
|
|
1403
|
+
if visible_source_context is None:
|
|
1404
|
+
if required or _visible_context_manifest(mapped, None):
|
|
1405
|
+
raise ProtocolError("Worker visible source context is required for its map")
|
|
1406
|
+
return None
|
|
1407
|
+
if not isinstance(visible_source_context, VisibleSourceContext):
|
|
1408
|
+
raise ProtocolError("Worker visible source context does not match its map")
|
|
1409
|
+
indices: dict[
|
|
1410
|
+
tuple[object, str, int],
|
|
1411
|
+
tuple[str, _ImmutableLineIndex],
|
|
1412
|
+
] = {}
|
|
1413
|
+
for unit in _referenced_visible_units(mapped):
|
|
1414
|
+
source_hash, length, line_starts = _visible_line_index_components(
|
|
1415
|
+
visible_source_context,
|
|
1416
|
+
unit,
|
|
1417
|
+
)
|
|
1418
|
+
indices[(unit.kind, unit.unit_id, unit.revision)] = (
|
|
1419
|
+
source_hash,
|
|
1420
|
+
_ImmutableLineIndex(length, line_starts),
|
|
1421
|
+
)
|
|
1422
|
+
context = _ImmutableVisibleSourceContext(indices)
|
|
1423
|
+
fingerprint = _visible_context_fingerprint(mapped, context)
|
|
1424
|
+
return _WorkerVisibleContextSnapshot(context, fingerprint)
|
|
1425
|
+
|
|
1426
|
+
|
|
1427
|
+
def _referenced_visible_units(mapped: MappedSource) -> tuple[SourceUnitRef, ...]:
|
|
1428
|
+
units: dict[tuple[object, str, int], SourceUnitRef] = {}
|
|
1429
|
+
compact_spans = getattr(mapped.source_map, "compact_visible_spans", None)
|
|
1430
|
+
references = (
|
|
1431
|
+
tuple(reference for _role, reference, _span in compact_spans())
|
|
1432
|
+
if compact_spans is not None
|
|
1433
|
+
else tuple(
|
|
1434
|
+
reference
|
|
1435
|
+
for segment in mapped.source_map.segments
|
|
1436
|
+
for reference in (segment.origin_ref, segment.anchor_ref)
|
|
1437
|
+
)
|
|
1438
|
+
)
|
|
1439
|
+
for reference in references:
|
|
1440
|
+
if reference is None:
|
|
1441
|
+
continue
|
|
1442
|
+
if not isinstance(reference, SourceUnitRef):
|
|
1443
|
+
raise ProtocolError(
|
|
1444
|
+
"Worker visible source context does not match its flattened map"
|
|
1445
|
+
)
|
|
1446
|
+
key = (reference.kind, reference.unit_id, reference.revision)
|
|
1447
|
+
existing = units.get(key)
|
|
1448
|
+
if existing is not None and existing.source_sha256 != reference.source_sha256:
|
|
1449
|
+
raise ProtocolError(
|
|
1450
|
+
"Worker visible source context has conflicting source identities"
|
|
1451
|
+
)
|
|
1452
|
+
units[key] = reference
|
|
1453
|
+
return tuple(
|
|
1454
|
+
sorted(
|
|
1455
|
+
units.values(),
|
|
1456
|
+
key=lambda unit: (
|
|
1457
|
+
unit.kind.value,
|
|
1458
|
+
unit.unit_id,
|
|
1459
|
+
unit.revision,
|
|
1460
|
+
unit.source_sha256,
|
|
1461
|
+
),
|
|
1462
|
+
)
|
|
1463
|
+
)
|
|
1464
|
+
|
|
1465
|
+
|
|
1466
|
+
def _visible_line_index_components(
|
|
1467
|
+
context: VisibleSourceContext,
|
|
1468
|
+
unit: SourceUnitRef,
|
|
1469
|
+
) -> tuple[str, int, tuple[int, ...]]:
|
|
1470
|
+
try:
|
|
1471
|
+
entry = context._indices.get( # type: ignore[attr-defined]
|
|
1472
|
+
(unit.kind, unit.unit_id, unit.revision)
|
|
1473
|
+
)
|
|
1474
|
+
source_hash, line_index = entry
|
|
1475
|
+
length = line_index._length
|
|
1476
|
+
line_starts = line_index._line_starts
|
|
1477
|
+
except (AttributeError, TypeError, ValueError):
|
|
1478
|
+
raise ProtocolError(
|
|
1479
|
+
"Worker visible source context does not cover every mapped unit"
|
|
1480
|
+
) from None
|
|
1481
|
+
if source_hash != unit.source_sha256 or type(line_starts) is not tuple:
|
|
1482
|
+
raise ProtocolError(
|
|
1483
|
+
"Worker visible source context does not match its mapped unit"
|
|
1484
|
+
)
|
|
1485
|
+
validated = _ImmutableLineIndex(length, line_starts)
|
|
1486
|
+
return source_hash, validated._length, validated._line_starts
|
|
1487
|
+
|
|
1488
|
+
|
|
1489
|
+
def _visible_context_manifest(
|
|
1490
|
+
mapped: MappedSource,
|
|
1491
|
+
context: VisibleSourceContext | None,
|
|
1492
|
+
) -> tuple[dict[str, object], ...]:
|
|
1493
|
+
entries: list[dict[str, object]] = []
|
|
1494
|
+
compact_spans = getattr(mapped.source_map, "compact_visible_spans", None)
|
|
1495
|
+
mapped_spans = (
|
|
1496
|
+
compact_spans()
|
|
1497
|
+
if compact_spans is not None
|
|
1498
|
+
else tuple(
|
|
1499
|
+
(role, reference, span)
|
|
1500
|
+
for segment in mapped.source_map.segments
|
|
1501
|
+
for role, reference, span in (
|
|
1502
|
+
("origin", segment.origin_ref, segment.origin),
|
|
1503
|
+
("anchor", segment.anchor_ref, segment.anchor_span),
|
|
1504
|
+
)
|
|
1505
|
+
if reference is not None and span is not None
|
|
1506
|
+
)
|
|
1507
|
+
)
|
|
1508
|
+
for segment_index, (role, reference, span) in enumerate(mapped_spans):
|
|
1509
|
+
if reference is None:
|
|
1510
|
+
continue
|
|
1511
|
+
if not isinstance(reference, SourceUnitRef) or span is None:
|
|
1512
|
+
raise ProtocolError(
|
|
1513
|
+
"Worker visible source context does not match its flattened map"
|
|
1514
|
+
)
|
|
1515
|
+
start = None if context is None else context.line_column(reference, span.start)
|
|
1516
|
+
end = None if context is None else context.line_column(reference, span.end)
|
|
1517
|
+
if context is not None and (start is None or end is None):
|
|
1518
|
+
raise ProtocolError(
|
|
1519
|
+
"Worker visible source context does not cover every mapped span"
|
|
1520
|
+
)
|
|
1521
|
+
entries.append(
|
|
1522
|
+
{
|
|
1523
|
+
"segment": segment_index,
|
|
1524
|
+
"role": role,
|
|
1525
|
+
"kind": reference.kind.value,
|
|
1526
|
+
"unit_id": reference.unit_id,
|
|
1527
|
+
"revision": reference.revision,
|
|
1528
|
+
"source_sha256": reference.source_sha256,
|
|
1529
|
+
"span": [span.start, span.end],
|
|
1530
|
+
"start": None if start is None else list(start),
|
|
1531
|
+
"end": None if end is None else list(end),
|
|
1532
|
+
}
|
|
1533
|
+
)
|
|
1534
|
+
return tuple(entries)
|
|
1535
|
+
|
|
1536
|
+
|
|
1537
|
+
def _visible_context_unit_manifest(
|
|
1538
|
+
mapped: MappedSource,
|
|
1539
|
+
context: VisibleSourceContext,
|
|
1540
|
+
) -> tuple[dict[str, object], ...]:
|
|
1541
|
+
entries: list[dict[str, object]] = []
|
|
1542
|
+
for unit in _referenced_visible_units(mapped):
|
|
1543
|
+
source_hash, length, line_starts = _visible_line_index_components(
|
|
1544
|
+
context,
|
|
1545
|
+
unit,
|
|
1546
|
+
)
|
|
1547
|
+
entries.append(
|
|
1548
|
+
{
|
|
1549
|
+
"kind": unit.kind.value,
|
|
1550
|
+
"unit_id": unit.unit_id,
|
|
1551
|
+
"revision": unit.revision,
|
|
1552
|
+
"source_sha256": source_hash,
|
|
1553
|
+
"character_length": length,
|
|
1554
|
+
"line_starts": list(line_starts),
|
|
1555
|
+
}
|
|
1556
|
+
)
|
|
1557
|
+
return tuple(entries)
|
|
1558
|
+
|
|
1559
|
+
|
|
1560
|
+
def _visible_context_fingerprint(
|
|
1561
|
+
mapped: MappedSource,
|
|
1562
|
+
context: VisibleSourceContext,
|
|
1563
|
+
) -> str:
|
|
1564
|
+
manifest = {
|
|
1565
|
+
"units": _visible_context_unit_manifest(mapped, context),
|
|
1566
|
+
"occurrences": _visible_context_manifest(mapped, context),
|
|
1567
|
+
}
|
|
1568
|
+
payload = json.dumps(
|
|
1569
|
+
manifest,
|
|
1570
|
+
ensure_ascii=False,
|
|
1571
|
+
sort_keys=True,
|
|
1572
|
+
separators=(",", ":"),
|
|
1573
|
+
)
|
|
1574
|
+
return sha256(payload.encode("utf-8")).hexdigest()
|
|
1575
|
+
|
|
1576
|
+
|
|
1577
|
+
def validate_production_worker_artifact(
|
|
1578
|
+
artifact: WorkerArtifact,
|
|
1579
|
+
) -> tuple[WorkerExport, ...]:
|
|
1580
|
+
"""Revalidate a complete production artifact immediately before activation."""
|
|
1581
|
+
return _validated_admitted_snapshot(artifact).catalog
|
|
1582
|
+
|
|
1583
|
+
|
|
1584
|
+
def _admission_contents(
|
|
1585
|
+
artifact: WorkerArtifact,
|
|
1586
|
+
) -> tuple[bytes, bytes, bytes, MappedSource]:
|
|
1587
|
+
capability = _worker_artifact_capability(artifact)
|
|
1588
|
+
assert capability is not None
|
|
1589
|
+
admission = capability.admission
|
|
1590
|
+
if not isinstance(admission, _WorkerArtifactAdmission):
|
|
1591
|
+
raise ProtocolError("Runtime API requires a proven worker artifact")
|
|
1592
|
+
return admission.contents(artifact)
|
|
1593
|
+
|
|
1594
|
+
|
|
1595
|
+
def _validated_admission_contents(
|
|
1596
|
+
artifact: WorkerArtifact,
|
|
1597
|
+
) -> tuple[bytes, bytes, bytes, MappedSource]:
|
|
1598
|
+
snapshot = _validated_admitted_snapshot(artifact)
|
|
1599
|
+
return (
|
|
1600
|
+
snapshot.source_bytes,
|
|
1601
|
+
snapshot.artifact_bytes,
|
|
1602
|
+
snapshot.proof,
|
|
1603
|
+
_snapshot_mapped_source(snapshot.mapped_source)[0],
|
|
1604
|
+
)
|
|
1605
|
+
|
|
1606
|
+
|
|
1607
|
+
def _validated_admitted_snapshot(
|
|
1608
|
+
artifact: WorkerArtifact,
|
|
1609
|
+
) -> _AdmittedWorkerSnapshot:
|
|
1610
|
+
capability = _worker_artifact_capability(artifact)
|
|
1611
|
+
assert capability is not None
|
|
1612
|
+
admission = capability.admission
|
|
1613
|
+
if not isinstance(admission, _WorkerArtifactAdmission):
|
|
1614
|
+
raise ProtocolError("Runtime API requires a proven worker artifact")
|
|
1615
|
+
snapshot = admission.snapshot(artifact)
|
|
1616
|
+
mapped = snapshot.mapped_source
|
|
1617
|
+
provenance = artifact.source_provenance
|
|
1618
|
+
if provenance is None or provenance.source_sha256 != artifact.source_sha256:
|
|
1619
|
+
raise ProtocolError("Worker artifact source map provenance is invalid")
|
|
1620
|
+
if mapped.artifact.kind is not SourceArtifactKind.WORKER_MODULE:
|
|
1621
|
+
raise ProtocolError("Worker artifact source map does not describe WORKER_MODULE")
|
|
1622
|
+
if (
|
|
1623
|
+
not snapshot.source_bytes
|
|
1624
|
+
or not snapshot.artifact_bytes
|
|
1625
|
+
or snapshot.source_sha256 != artifact.source_sha256
|
|
1626
|
+
or snapshot.artifact_sha256 != artifact.artifact_sha256
|
|
1627
|
+
or mapped.artifact.source_sha256 != artifact.source_sha256
|
|
1628
|
+
):
|
|
1629
|
+
raise ProtocolError("Worker artifact source map source hash does not match")
|
|
1630
|
+
if snapshot.source_map_sha256 != provenance.source_map_sha256:
|
|
1631
|
+
raise ProtocolError("Worker artifact source map hash does not match")
|
|
1632
|
+
if (
|
|
1633
|
+
provenance.worker_generation != snapshot.worker_generation
|
|
1634
|
+
or provenance.worker_manifest_sha256
|
|
1635
|
+
!= snapshot.worker_manifest_sha256
|
|
1636
|
+
):
|
|
1637
|
+
raise ProtocolError("Worker artifact source map provenance is invalid")
|
|
1638
|
+
if (
|
|
1639
|
+
capability.source_path != snapshot.source_path
|
|
1640
|
+
or capability.artifact_path != snapshot.artifact_path
|
|
1641
|
+
or capability.expected_version != snapshot.expected_version
|
|
1642
|
+
or capability.expected_value != snapshot.expected_value
|
|
1643
|
+
or snapshot.mapped_guard != _mapped_source_guard(mapped)
|
|
1644
|
+
):
|
|
1645
|
+
raise ProtocolError("Worker artifact source map proof does not match admission")
|
|
1646
|
+
context_snapshot = snapshot.visible_context_snapshot
|
|
1647
|
+
if context_snapshot is None:
|
|
1648
|
+
if _visible_context_manifest(mapped, None):
|
|
1649
|
+
raise ProtocolError("Worker visible source context is missing from admission")
|
|
1650
|
+
if snapshot.visible_context_fingerprint or snapshot.visible_context_guard:
|
|
1651
|
+
raise ProtocolError("Worker visible source context fingerprint does not match")
|
|
1652
|
+
else:
|
|
1653
|
+
if (
|
|
1654
|
+
not compare_digest(
|
|
1655
|
+
snapshot.visible_context_fingerprint,
|
|
1656
|
+
context_snapshot.fingerprint,
|
|
1657
|
+
)
|
|
1658
|
+
or snapshot.visible_context_guard
|
|
1659
|
+
!= _visible_context_snapshot_guard(context_snapshot)
|
|
1660
|
+
):
|
|
1661
|
+
raise ProtocolError("Worker visible source context fingerprint does not match")
|
|
1662
|
+
if not compare_digest(
|
|
1663
|
+
snapshot.proof,
|
|
1664
|
+
_artifact_proof(artifact, snapshot),
|
|
1665
|
+
):
|
|
1666
|
+
raise ProtocolError("Worker artifact source map proof does not match admission")
|
|
1667
|
+
if admission.catalog(artifact) != snapshot.catalog:
|
|
1668
|
+
raise ProtocolError("Runtime API requires a proven worker artifact")
|
|
1669
|
+
return snapshot
|
|
1670
|
+
|
|
1671
|
+
|
|
1672
|
+
def _validate_artifact_files(
|
|
1673
|
+
source_path: Path,
|
|
1674
|
+
artifact_path: Path,
|
|
1675
|
+
exports: tuple[WorkerExport, ...],
|
|
1676
|
+
*,
|
|
1677
|
+
source_validation: _WorkerSourceCatalogValidation | None = None,
|
|
1678
|
+
mapped_source: MappedSource | None = None,
|
|
1679
|
+
) -> tuple[
|
|
1680
|
+
tuple[WorkerExport, ...],
|
|
1681
|
+
bytes,
|
|
1682
|
+
bytes,
|
|
1683
|
+
str,
|
|
1684
|
+
MappedSource | None,
|
|
1685
|
+
]:
|
|
1686
|
+
catalog = validate_worker_export_catalog(exports, require_nonempty=True)
|
|
1687
|
+
if not source_path.is_file():
|
|
1688
|
+
raise ProtocolError("Worker source file does not exist")
|
|
1689
|
+
if not artifact_path.is_file():
|
|
1690
|
+
raise ProtocolError("Worker artifact file does not exist")
|
|
1691
|
+
if source_validation is None:
|
|
1692
|
+
source_bytes = source_path.read_bytes()
|
|
1693
|
+
catalog = _validate_source_catalog(source_path, source_bytes, exports)
|
|
1694
|
+
source_hash = sha256(source_bytes).hexdigest()
|
|
1695
|
+
mapped_snapshot = None
|
|
1696
|
+
elif isinstance(source_validation, _WorkerSourceCatalogValidation):
|
|
1697
|
+
catalog, source_bytes, source_hash, mapped_snapshot = (
|
|
1698
|
+
_reuse_worker_source_catalog(
|
|
1699
|
+
source_validation,
|
|
1700
|
+
source_path,
|
|
1701
|
+
exports,
|
|
1702
|
+
mapped_source,
|
|
1703
|
+
)
|
|
1704
|
+
)
|
|
1705
|
+
else:
|
|
1706
|
+
raise ProtocolError("Worker source catalog validation is invalid")
|
|
1707
|
+
return (
|
|
1708
|
+
catalog,
|
|
1709
|
+
source_bytes,
|
|
1710
|
+
artifact_path.read_bytes(),
|
|
1711
|
+
source_hash,
|
|
1712
|
+
mapped_snapshot,
|
|
1713
|
+
)
|
|
1714
|
+
|
|
1715
|
+
|
|
1716
|
+
def _validate_source_catalog(
|
|
1717
|
+
source_path: Path,
|
|
1718
|
+
source_bytes: bytes,
|
|
1719
|
+
exports: tuple[WorkerExport, ...],
|
|
1720
|
+
) -> tuple[WorkerExport, ...]:
|
|
1721
|
+
catalog = validate_worker_export_catalog(exports, require_nonempty=True)
|
|
1722
|
+
source = source_bytes.decode("utf-8")
|
|
1723
|
+
binding = SemanticNotebookLowerer(PythonParserTarget.from_generated()).bind_module(source)
|
|
1724
|
+
exported = {name.casefold() for name in binding.exported_method_names}
|
|
1725
|
+
for export in catalog:
|
|
1726
|
+
if export.method.casefold() not in exported:
|
|
1727
|
+
raise ProtocolError(
|
|
1728
|
+
f"Worker catalog method {export.method!r} is not exported by {source_path}"
|
|
1729
|
+
)
|
|
1730
|
+
return catalog
|
|
1731
|
+
|
|
1732
|
+
|
|
1733
|
+
def _artifact_proof(
|
|
1734
|
+
artifact: WorkerArtifact,
|
|
1735
|
+
snapshot: _AdmittedWorkerSnapshot,
|
|
1736
|
+
) -> bytes:
|
|
1737
|
+
fields = (
|
|
1738
|
+
artifact.logical_name,
|
|
1739
|
+
str(snapshot.artifact_path),
|
|
1740
|
+
str(snapshot.source_path),
|
|
1741
|
+
artifact.source_sha256,
|
|
1742
|
+
"" if artifact.source_map_sha256 is None else artifact.source_map_sha256,
|
|
1743
|
+
(
|
|
1744
|
+
""
|
|
1745
|
+
if artifact.source_provenance is None
|
|
1746
|
+
or artifact.source_provenance.worker_generation is None
|
|
1747
|
+
else str(artifact.source_provenance.worker_generation)
|
|
1748
|
+
),
|
|
1749
|
+
(
|
|
1750
|
+
""
|
|
1751
|
+
if artifact.source_provenance is None
|
|
1752
|
+
or artifact.source_provenance.worker_manifest_sha256 is None
|
|
1753
|
+
else artifact.source_provenance.worker_manifest_sha256
|
|
1754
|
+
),
|
|
1755
|
+
artifact.artifact_sha256,
|
|
1756
|
+
"" if snapshot.expected_version is None else snapshot.expected_version,
|
|
1757
|
+
str(snapshot.expected_value),
|
|
1758
|
+
snapshot.source_sha256,
|
|
1759
|
+
snapshot.artifact_sha256,
|
|
1760
|
+
snapshot.source_map_sha256,
|
|
1761
|
+
snapshot.visible_context_fingerprint,
|
|
1762
|
+
str(id(snapshot.binary)),
|
|
1763
|
+
str(id(snapshot.source_bytes)),
|
|
1764
|
+
str(id(snapshot.artifact_bytes)),
|
|
1765
|
+
repr(snapshot.mapped_guard),
|
|
1766
|
+
repr(snapshot.visible_context_guard),
|
|
1767
|
+
str(id(snapshot.catalog)),
|
|
1768
|
+
*(
|
|
1769
|
+
f"{item.public_path}\0{item.method}\0{item.receiver_module or ''}"
|
|
1770
|
+
for item in artifact.exports
|
|
1771
|
+
),
|
|
1772
|
+
)
|
|
1773
|
+
return digest(_ARTIFACT_PROOF_KEY, "\x1f".join(fields).encode("utf-8"), "sha256")
|
|
1774
|
+
|
|
1775
|
+
|
|
1776
|
+
def stage_worker_module_instruction(
|
|
1777
|
+
content: bytes,
|
|
1778
|
+
*,
|
|
1779
|
+
logical_name: str,
|
|
1780
|
+
artifact_sha256: str,
|
|
1781
|
+
registration_name: str,
|
|
1782
|
+
) -> str:
|
|
1783
|
+
"""Upload and connect one module without touching the active generation."""
|
|
1784
|
+
if not isinstance(content, bytes) or not content:
|
|
1785
|
+
raise ValueError("worker artifact content must be nonempty bytes")
|
|
1786
|
+
if (
|
|
1787
|
+
not _worker_safe_identifier(logical_name)
|
|
1788
|
+
or fullmatch(r"[0-9a-f]{64}", artifact_sha256) is None
|
|
1789
|
+
or not _worker_safe_identifier(registration_name)
|
|
1790
|
+
):
|
|
1791
|
+
raise ValueError("worker artifact stage identity is invalid")
|
|
1792
|
+
encoded = bsl_string_literal(b64encode(content).decode("ascii"))
|
|
1793
|
+
name = bsl_string_literal(registration_name)
|
|
1794
|
+
logical_digest = sha256(logical_name.casefold().encode("utf-8")).hexdigest()
|
|
1795
|
+
marker = bsl_string_literal(
|
|
1796
|
+
_worker_artifact_stage_marker(
|
|
1797
|
+
artifact_sha256=artifact_sha256,
|
|
1798
|
+
logical_name_sha256=logical_digest,
|
|
1799
|
+
phase="connect",
|
|
1800
|
+
)
|
|
1801
|
+
)
|
|
1802
|
+
return "\n".join(
|
|
1803
|
+
(
|
|
1804
|
+
f"МаркерАртефактаWorker = {marker};",
|
|
1805
|
+
"Попытка",
|
|
1806
|
+
' ЭтапАртефактаWorker = "upload";',
|
|
1807
|
+
' АдресАртефактаWorker = ПоместитьВоВременноеХранилище('
|
|
1808
|
+
f"Base64Значение({encoded}));",
|
|
1809
|
+
' ЭтапАртефактаWorker = "connect";',
|
|
1810
|
+
" ИмяАртефактаWorker = ВнешниеОбработки.Подключить(",
|
|
1811
|
+
f" АдресАртефактаWorker, {name}, Ложь);",
|
|
1812
|
+
"Исключение",
|
|
1813
|
+
" ОшибкаАртефактаWorker = ОписаниеОшибки();",
|
|
1814
|
+
" ВызватьИсключение МаркерАртефактаWorker",
|
|
1815
|
+
' + ";boundary=" + ЭтапАртефактаWorker',
|
|
1816
|
+
" + Символы.ПС + ОшибкаАртефактаWorker;",
|
|
1817
|
+
"КонецПопытки;",
|
|
1818
|
+
"Результат = ИмяАртефактаWorker;",
|
|
1819
|
+
)
|
|
1820
|
+
)
|
|
1821
|
+
|
|
1822
|
+
|
|
1823
|
+
_WORKER_ARTIFACT_STAGE_PREFIX = "onec-worker-artifact-stage="
|
|
1824
|
+
|
|
1825
|
+
|
|
1826
|
+
def _worker_artifact_stage_marker(
|
|
1827
|
+
*,
|
|
1828
|
+
artifact_sha256: str,
|
|
1829
|
+
logical_name_sha256: str,
|
|
1830
|
+
phase: Literal["connect"],
|
|
1831
|
+
) -> str:
|
|
1832
|
+
return (
|
|
1833
|
+
f"{_WORKER_ARTIFACT_STAGE_PREFIX}"
|
|
1834
|
+
f"artifact_sha256={artifact_sha256};"
|
|
1835
|
+
f"logical_name_sha256={logical_name_sha256};phase={phase}"
|
|
1836
|
+
)
|
|
1837
|
+
|
|
1838
|
+
|
|
1839
|
+
def _worker_safe_identifier(value: object) -> bool:
|
|
1840
|
+
return _worker_stage_safe_identifier(value)
|
|
1841
|
+
|
|
1842
|
+
|
|
1843
|
+
def _worker_reload_platform_message(error: BaseException) -> str:
|
|
1844
|
+
"""Remove our control header without changing the platform diagnostic body."""
|
|
1845
|
+
message = str(error)
|
|
1846
|
+
artifact_marker = message.find(_WORKER_ARTIFACT_STAGE_PREFIX)
|
|
1847
|
+
if artifact_marker >= 0:
|
|
1848
|
+
artifact_line = message[
|
|
1849
|
+
artifact_marker + len(_WORKER_ARTIFACT_STAGE_PREFIX) :
|
|
1850
|
+
]
|
|
1851
|
+
header, separator, remainder = artifact_line.partition("\n")
|
|
1852
|
+
if separator and _worker_artifact_stage_header(header.rstrip("\r")) is not None:
|
|
1853
|
+
length = search(r";diagnostic_utf16_length=([1-9][0-9]{0,5})\r?$", header)
|
|
1854
|
+
if length is not None:
|
|
1855
|
+
# 1C appends the outer Raise stack after this UTF-16-sized body.
|
|
1856
|
+
units = int(length.group(1))
|
|
1857
|
+
encoded = remainder[:units].encode("utf-16-le")
|
|
1858
|
+
if len(encoded) < units * 2:
|
|
1859
|
+
return message
|
|
1860
|
+
try:
|
|
1861
|
+
return encoded[:units * 2].decode("utf-16-le")
|
|
1862
|
+
except UnicodeDecodeError:
|
|
1863
|
+
return message
|
|
1864
|
+
return remainder
|
|
1865
|
+
return message
|
|
1866
|
+
|
|
1867
|
+
|
|
1868
|
+
def _worker_artifact_stage_header(
|
|
1869
|
+
value: str,
|
|
1870
|
+
) -> tuple[str, str, Literal["upload", "connect", "create"]] | None:
|
|
1871
|
+
match = fullmatch(
|
|
1872
|
+
r"artifact_sha256=([0-9a-f]{64});"
|
|
1873
|
+
r"logical_name_sha256=([0-9a-f]{64});"
|
|
1874
|
+
r"(?:phase=connect;boundary=(upload|connect)|phase=create;boundary=(create)"
|
|
1875
|
+
r"(?:;diagnostic_utf16_length=[1-9][0-9]{0,5})?)",
|
|
1876
|
+
value,
|
|
1877
|
+
)
|
|
1878
|
+
if match is None:
|
|
1879
|
+
return None
|
|
1880
|
+
artifact_sha256, logical_name_sha256, boundary, create = match.groups()
|
|
1881
|
+
return (
|
|
1882
|
+
artifact_sha256,
|
|
1883
|
+
logical_name_sha256,
|
|
1884
|
+
cast(Literal["upload", "connect", "create"], boundary or create),
|
|
1885
|
+
)
|
|
1886
|
+
|
|
1887
|
+
|
|
1888
|
+
def _worker_artifact_failure_identity(
|
|
1889
|
+
error: BaseException,
|
|
1890
|
+
) -> tuple[str, str, Literal["upload", "connect", "create"]] | None:
|
|
1891
|
+
message = str(error)
|
|
1892
|
+
marker = message.find(_WORKER_ARTIFACT_STAGE_PREFIX)
|
|
1893
|
+
if marker < 0:
|
|
1894
|
+
return None
|
|
1895
|
+
value = message[marker + len(_WORKER_ARTIFACT_STAGE_PREFIX) :].splitlines()[0]
|
|
1896
|
+
return _worker_artifact_stage_header(value.rstrip("\r"))
|
|
1897
|
+
|
|
1898
|
+
|
|
1899
|
+
def worker_artifact_stage_failure(
|
|
1900
|
+
error: BaseException,
|
|
1901
|
+
) -> WorkerArtifactStageFailure | None:
|
|
1902
|
+
"""Return the exact admitted artifact/phase marker without diagnostic prose."""
|
|
1903
|
+
identity = _worker_artifact_failure_identity(error)
|
|
1904
|
+
if identity is None:
|
|
1905
|
+
return None
|
|
1906
|
+
artifact_sha256, _logical_name_sha256, phase = identity
|
|
1907
|
+
return WorkerArtifactStageFailure(artifact_sha256, phase)
|
|
1908
|
+
|
|
1909
|
+
|
|
1910
|
+
def _worker_artifact_diagnostic_source_from_snapshot(
|
|
1911
|
+
artifact: WorkerArtifact,
|
|
1912
|
+
snapshot: _AdmittedWorkerSnapshot,
|
|
1913
|
+
*,
|
|
1914
|
+
logical_name: str,
|
|
1915
|
+
revision: int,
|
|
1916
|
+
registration_name: str,
|
|
1917
|
+
manifest_sha256: str,
|
|
1918
|
+
artifact_sha256: str,
|
|
1919
|
+
source_map_sha256: str,
|
|
1920
|
+
) -> WorkerDiagnosticArtifact:
|
|
1921
|
+
"""Bind one already-validated admitted snapshot without cloning it."""
|
|
1922
|
+
mapped = snapshot.mapped_source
|
|
1923
|
+
visible_context_snapshot = snapshot.visible_context_snapshot
|
|
1924
|
+
if (
|
|
1925
|
+
not isinstance(artifact, WorkerArtifact)
|
|
1926
|
+
or not isinstance(snapshot, _AdmittedWorkerSnapshot)
|
|
1927
|
+
or artifact.logical_name.casefold() != logical_name.casefold()
|
|
1928
|
+
or artifact.source_sha256 != snapshot.source_sha256
|
|
1929
|
+
or artifact.artifact_sha256 != artifact_sha256
|
|
1930
|
+
or snapshot.artifact_sha256 != artifact_sha256
|
|
1931
|
+
or artifact.source_map_sha256 != source_map_sha256
|
|
1932
|
+
or snapshot.source_map_sha256 != source_map_sha256
|
|
1933
|
+
or mapped.artifact.source_sha256 != snapshot.source_sha256
|
|
1934
|
+
or mapped.source_map_sha256 != source_map_sha256
|
|
1935
|
+
or (
|
|
1936
|
+
visible_context_snapshot is not None
|
|
1937
|
+
and not isinstance(
|
|
1938
|
+
visible_context_snapshot.context,
|
|
1939
|
+
_ImmutableVisibleSourceContext,
|
|
1940
|
+
)
|
|
1941
|
+
)
|
|
1942
|
+
):
|
|
1943
|
+
raise ProtocolError("Worker diagnostic source identity is invalid")
|
|
1944
|
+
try:
|
|
1945
|
+
return WorkerDiagnosticArtifact(
|
|
1946
|
+
logical_name=logical_name,
|
|
1947
|
+
revision=revision,
|
|
1948
|
+
artifact_sha256=artifact_sha256,
|
|
1949
|
+
registration_name=registration_name,
|
|
1950
|
+
manifest_sha256=manifest_sha256,
|
|
1951
|
+
source_map_sha256=source_map_sha256,
|
|
1952
|
+
mapped_source=mapped,
|
|
1953
|
+
visible_source_context=(
|
|
1954
|
+
None
|
|
1955
|
+
if visible_context_snapshot is None
|
|
1956
|
+
else visible_context_snapshot.context
|
|
1957
|
+
),
|
|
1958
|
+
)
|
|
1959
|
+
except ValueError:
|
|
1960
|
+
raise ProtocolError("Worker diagnostic source identity is invalid") from None
|
|
1961
|
+
|
|
1962
|
+
|
|
1963
|
+
def worker_artifact_diagnostic_source(
|
|
1964
|
+
artifact: WorkerArtifact,
|
|
1965
|
+
*,
|
|
1966
|
+
logical_name: str,
|
|
1967
|
+
revision: int,
|
|
1968
|
+
registration_name: str,
|
|
1969
|
+
manifest_sha256: str,
|
|
1970
|
+
artifact_sha256: str,
|
|
1971
|
+
source_map_sha256: str,
|
|
1972
|
+
) -> WorkerDiagnosticArtifact:
|
|
1973
|
+
"""Expose an admitted source map as a private, manifest-fenced descriptor."""
|
|
1974
|
+
snapshot = _validated_admitted_snapshot(artifact)
|
|
1975
|
+
mapped = _snapshot_mapped_source(snapshot.mapped_source)[0]
|
|
1976
|
+
visible_context_snapshot = _clone_visible_context_snapshot(
|
|
1977
|
+
snapshot.visible_context_snapshot
|
|
1978
|
+
)
|
|
1979
|
+
if (
|
|
1980
|
+
artifact.artifact_sha256 != artifact_sha256
|
|
1981
|
+
or artifact.source_map_sha256 != source_map_sha256
|
|
1982
|
+
or snapshot.source_map_sha256 != source_map_sha256
|
|
1983
|
+
):
|
|
1984
|
+
raise ProtocolError("Worker diagnostic source identity is invalid")
|
|
1985
|
+
try:
|
|
1986
|
+
return WorkerDiagnosticArtifact(
|
|
1987
|
+
logical_name=logical_name,
|
|
1988
|
+
revision=revision,
|
|
1989
|
+
artifact_sha256=artifact_sha256,
|
|
1990
|
+
registration_name=registration_name,
|
|
1991
|
+
manifest_sha256=manifest_sha256,
|
|
1992
|
+
source_map_sha256=source_map_sha256,
|
|
1993
|
+
mapped_source=mapped,
|
|
1994
|
+
visible_source_context=(
|
|
1995
|
+
None
|
|
1996
|
+
if visible_context_snapshot is None
|
|
1997
|
+
else visible_context_snapshot.context
|
|
1998
|
+
),
|
|
1999
|
+
)
|
|
2000
|
+
except ValueError:
|
|
2001
|
+
raise ProtocolError("Worker diagnostic source identity is invalid") from None
|
|
2002
|
+
|
|
2003
|
+
|
|
2004
|
+
def remap_worker_artifact_stage_error(
|
|
2005
|
+
error: BslExecutionError,
|
|
2006
|
+
*,
|
|
2007
|
+
candidate_manifest_sha256: str,
|
|
2008
|
+
candidate_artifacts: tuple[WorkerDiagnosticArtifact, ...],
|
|
2009
|
+
) -> BslExecutionError:
|
|
2010
|
+
"""Attach exact or explicitly-unmapped evidence to one stage failure."""
|
|
2011
|
+
identity = worker_artifact_stage_failure(error)
|
|
2012
|
+
if identity is None:
|
|
2013
|
+
return error
|
|
2014
|
+
parsed = parse_platform_diagnostic(_worker_reload_platform_message(error))
|
|
2015
|
+
if identity.phase == "create":
|
|
2016
|
+
header = _worker_artifact_failure_identity(error)
|
|
2017
|
+
assert header is not None
|
|
2018
|
+
candidate_artifacts = tuple(
|
|
2019
|
+
artifact for artifact in candidate_artifacts
|
|
2020
|
+
if sha256(artifact.logical_name.casefold().encode("utf-8")).hexdigest()
|
|
2021
|
+
== header[1]
|
|
2022
|
+
)
|
|
2023
|
+
diagnostic = remap_worker_stage_diagnostic(
|
|
2024
|
+
parsed,
|
|
2025
|
+
artifact_sha256=identity.artifact_sha256,
|
|
2026
|
+
phase=identity.phase,
|
|
2027
|
+
candidate_manifest_sha256=candidate_manifest_sha256,
|
|
2028
|
+
candidate_artifacts=candidate_artifacts,
|
|
2029
|
+
)
|
|
2030
|
+
return BslExecutionError(
|
|
2031
|
+
str(error),
|
|
2032
|
+
messages=error.messages,
|
|
2033
|
+
diagnostic=diagnostic,
|
|
2034
|
+
)
|