amkernel 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- amkernel/__init__.py +7 -0
- amkernel/__main__.py +5 -0
- amkernel/artifact_store.py +1363 -0
- amkernel/cli.py +172 -0
- amkernel/distribution.py +597 -0
- amkernel/output.py +444 -0
- amkernel/protocol.py +1464 -0
- amkernel/py.typed +0 -0
- amkernel/runner.py +1222 -0
- amkernel/runtime.py +581 -0
- amkernel/server.py +774 -0
- amkernel/skill/SKILL.md +124 -0
- amkernel/skill/references/amkernel-appium.md +46 -0
- amkernel/skill/references/official-skills.json +232 -0
- amkernel/skill/references/official-sources.json +21 -0
- amkernel/skill/scripts/verify_official_skills.py +252 -0
- amkernel/structured_output.py +328 -0
- amkernel/supervisor.py +1453 -0
- amkernel/workspace.py +305 -0
- amkernel-0.1.0.dist-info/METADATA +105 -0
- amkernel-0.1.0.dist-info/RECORD +23 -0
- amkernel-0.1.0.dist-info/WHEEL +4 -0
- amkernel-0.1.0.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,1363 @@
|
|
|
1
|
+
"""Bounded process-scoped storage for immutable execution artifacts."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import stat
|
|
10
|
+
import tempfile
|
|
11
|
+
import uuid
|
|
12
|
+
from collections import OrderedDict
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Final, Literal
|
|
17
|
+
|
|
18
|
+
from amkernel.protocol import (
|
|
19
|
+
ARTIFACT_RESOURCE_PART_BYTES,
|
|
20
|
+
ARTIFACT_STORE_QUOTA_BYTES,
|
|
21
|
+
EXECUTION_BUNDLE_RESERVATION_BYTES,
|
|
22
|
+
MAX_ARTIFACT_BYTES,
|
|
23
|
+
MAX_ARTIFACT_PARTS,
|
|
24
|
+
MAX_EXECUTION_PARTS,
|
|
25
|
+
MAX_STORED_ITEMS,
|
|
26
|
+
SAFE_MIME_TYPES,
|
|
27
|
+
validate_logical_name,
|
|
28
|
+
validate_mime_type,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
_ITEM_CHARGE: Final = 4_096
|
|
32
|
+
_PART_CHARGE: Final = 4_096
|
|
33
|
+
_MANIFEST_MIME: Final = "application/vnd.amkernel.artifact-manifest+json"
|
|
34
|
+
_URI_PREFIX: Final = "amkernel://artifact/"
|
|
35
|
+
_MAX_MANIFEST_BYTES: Final = 65_536
|
|
36
|
+
_ID_ATTEMPTS: Final = 3
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class ArtifactStoreError(RuntimeError):
|
|
40
|
+
"""Artifact storage, quota, integrity, or cleanup failed."""
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class ArtifactNotFoundError(ArtifactStoreError):
|
|
44
|
+
"""A resource URI is malformed, foreign, or no longer retained."""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class ArtifactIntegrityError(ArtifactStoreError):
|
|
48
|
+
"""Retained bytes no longer match their immutable metadata."""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass(frozen=True, slots=True)
|
|
52
|
+
class ArtifactRecord:
|
|
53
|
+
"""Path-free immutable metadata published to callers."""
|
|
54
|
+
|
|
55
|
+
uri: str
|
|
56
|
+
name: str
|
|
57
|
+
declared_mime_type: str
|
|
58
|
+
mime_type: str
|
|
59
|
+
representation: Literal["raw", "manifest"]
|
|
60
|
+
size: int
|
|
61
|
+
sha256: str
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True, slots=True)
|
|
65
|
+
class ArtifactResource:
|
|
66
|
+
"""One bounded, verified resource response."""
|
|
67
|
+
|
|
68
|
+
uri: str
|
|
69
|
+
mime_type: str
|
|
70
|
+
data: bytes
|
|
71
|
+
text: bool
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
_Identity = tuple[int, int]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _identity(observed: os.stat_result) -> _Identity:
|
|
78
|
+
return observed.st_dev, observed.st_ino
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass(frozen=True, slots=True)
|
|
82
|
+
class _Part:
|
|
83
|
+
name: str
|
|
84
|
+
offset: int
|
|
85
|
+
size: int
|
|
86
|
+
sha256: str
|
|
87
|
+
identity: _Identity
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@dataclass(frozen=True, slots=True)
|
|
91
|
+
class _Artifact:
|
|
92
|
+
record: ArtifactRecord
|
|
93
|
+
directory: str
|
|
94
|
+
directory_identity: _Identity
|
|
95
|
+
parts: tuple[_Part, ...]
|
|
96
|
+
manifest_name: str | None
|
|
97
|
+
manifest_size: int | None
|
|
98
|
+
manifest_sha256: str | None
|
|
99
|
+
manifest_identity: _Identity | None
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@dataclass(slots=True)
|
|
103
|
+
class _Bundle:
|
|
104
|
+
identifier: str
|
|
105
|
+
directory: str
|
|
106
|
+
artifacts: dict[str, _Artifact]
|
|
107
|
+
directory_identity: _Identity
|
|
108
|
+
charge: int
|
|
109
|
+
leases: int = 0
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@dataclass(slots=True)
|
|
113
|
+
class _DeletionProgress:
|
|
114
|
+
tombstones: dict[str, str] = field(default_factory=dict)
|
|
115
|
+
completed: set[str] = field(default_factory=set)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@dataclass(slots=True)
|
|
119
|
+
class _DeletingBundle:
|
|
120
|
+
bundle: _Bundle
|
|
121
|
+
progress: _DeletionProgress = field(default_factory=_DeletionProgress)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
IdFactory = Callable[[], str]
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _new_id() -> str:
|
|
128
|
+
return uuid.uuid4().hex
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _valid_id(value: str) -> bool:
|
|
132
|
+
try:
|
|
133
|
+
parsed = uuid.UUID(hex=value)
|
|
134
|
+
except (ValueError, AttributeError):
|
|
135
|
+
return False
|
|
136
|
+
return parsed.version == 4 and parsed.hex == value
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _safe_mime(declared: str) -> str:
|
|
140
|
+
return declared if declared in SAFE_MIME_TYPES else "application/octet-stream"
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class ArtifactStore:
|
|
144
|
+
"""Own one private root and serve bounded immutable artifact resources."""
|
|
145
|
+
|
|
146
|
+
def __init__(
|
|
147
|
+
self,
|
|
148
|
+
*,
|
|
149
|
+
temporary_parent: Path | None = None,
|
|
150
|
+
id_factory: IdFactory = _new_id,
|
|
151
|
+
quota_bytes: int = ARTIFACT_STORE_QUOTA_BYTES,
|
|
152
|
+
reservation_bytes: int = EXECUTION_BUNDLE_RESERVATION_BYTES,
|
|
153
|
+
) -> None:
|
|
154
|
+
if quota_bytes < reservation_bytes:
|
|
155
|
+
raise ValueError("artifact quota must fit one execution reservation")
|
|
156
|
+
if os.name != "posix":
|
|
157
|
+
raise ArtifactStoreError("Artifact Store requires POSIX descriptor operations")
|
|
158
|
+
self._id_factory = id_factory
|
|
159
|
+
self._quota_bytes = quota_bytes
|
|
160
|
+
self._reservation_bytes = reservation_bytes
|
|
161
|
+
self._parent = Path(temporary_parent or tempfile.gettempdir())
|
|
162
|
+
self._parent_fd = os.open(self._parent, os.O_RDONLY | os.O_DIRECTORY)
|
|
163
|
+
self._server_id, self._root_name = self._create_root()
|
|
164
|
+
nofollow = getattr(os, "O_NOFOLLOW", 0)
|
|
165
|
+
self._root_fd = os.open(
|
|
166
|
+
self._root_name,
|
|
167
|
+
os.O_RDONLY | os.O_DIRECTORY | nofollow,
|
|
168
|
+
dir_fd=self._parent_fd,
|
|
169
|
+
)
|
|
170
|
+
root_stat = os.fstat(self._root_fd)
|
|
171
|
+
self._root_identity = _identity(root_stat)
|
|
172
|
+
os.fchmod(self._root_fd, 0o700)
|
|
173
|
+
self._deleting_bundles: dict[str, _DeletingBundle] = {}
|
|
174
|
+
self._bundles: OrderedDict[str, _Bundle] = OrderedDict()
|
|
175
|
+
self._artifacts: dict[str, tuple[_Bundle, _Artifact]] = {}
|
|
176
|
+
self._reserved_artifact_ids: set[str] = set()
|
|
177
|
+
self._active: set[ArtifactBundle] = set()
|
|
178
|
+
self._retained_charge = 0
|
|
179
|
+
self._reserved_charge = 0
|
|
180
|
+
self._gate = asyncio.Lock()
|
|
181
|
+
self._read_admission = asyncio.Semaphore(1)
|
|
182
|
+
self._closed = False
|
|
183
|
+
self._cleanup_complete = False
|
|
184
|
+
self._cleanup_name = self._root_name
|
|
185
|
+
|
|
186
|
+
@property
|
|
187
|
+
def server_id(self) -> str:
|
|
188
|
+
return self._server_id
|
|
189
|
+
|
|
190
|
+
def _create_root(self) -> tuple[str, str]:
|
|
191
|
+
for _ in range(_ID_ATTEMPTS):
|
|
192
|
+
identifier = self._id_factory()
|
|
193
|
+
if not _valid_id(identifier):
|
|
194
|
+
raise ArtifactStoreError("ID factory returned an invalid UUIDv4")
|
|
195
|
+
name = f"amkernel-{identifier}"
|
|
196
|
+
try:
|
|
197
|
+
os.mkdir(name, mode=0o700, dir_fd=self._parent_fd)
|
|
198
|
+
except FileExistsError:
|
|
199
|
+
continue
|
|
200
|
+
return identifier, name
|
|
201
|
+
os.close(self._parent_fd)
|
|
202
|
+
raise ArtifactStoreError("Artifact Store ID collision exhaustion")
|
|
203
|
+
|
|
204
|
+
async def begin_bundle(self) -> ArtifactBundle:
|
|
205
|
+
"""Reserve exact call capacity and return one unpublished bundle."""
|
|
206
|
+
async with self._read_admission:
|
|
207
|
+
async with self._gate:
|
|
208
|
+
self._require_open()
|
|
209
|
+
self._retry_deleting_bundles()
|
|
210
|
+
while (
|
|
211
|
+
self._retained_charge + self._reserved_charge + self._reservation_bytes
|
|
212
|
+
> self._quota_bytes
|
|
213
|
+
):
|
|
214
|
+
if not self._evict_oldest_unleased():
|
|
215
|
+
raise ArtifactStoreError("Artifact Store quota is unavailable")
|
|
216
|
+
identifier, stage_name, stage_identity = self._create_bundle_directory()
|
|
217
|
+
bundle = ArtifactBundle(self, identifier, stage_name, stage_identity)
|
|
218
|
+
self._active.add(bundle)
|
|
219
|
+
self._reserved_charge += self._reservation_bytes
|
|
220
|
+
return bundle
|
|
221
|
+
|
|
222
|
+
def _create_bundle_directory(self) -> tuple[str, str, _Identity]:
|
|
223
|
+
for _ in range(_ID_ATTEMPTS):
|
|
224
|
+
identifier = self._id_factory()
|
|
225
|
+
if not _valid_id(identifier):
|
|
226
|
+
raise ArtifactStoreError("ID factory returned an invalid UUIDv4")
|
|
227
|
+
name = f"stage-{identifier}"
|
|
228
|
+
try:
|
|
229
|
+
os.mkdir(name, mode=0o700, dir_fd=self._root_fd)
|
|
230
|
+
except FileExistsError:
|
|
231
|
+
continue
|
|
232
|
+
os.chmod(name, 0o700, dir_fd=self._root_fd, follow_symlinks=False)
|
|
233
|
+
observed = os.stat(name, dir_fd=self._root_fd, follow_symlinks=False)
|
|
234
|
+
return identifier, name, _identity(observed)
|
|
235
|
+
raise ArtifactStoreError("bundle ID collision exhaustion")
|
|
236
|
+
|
|
237
|
+
async def read(self, uri: str) -> ArtifactResource:
|
|
238
|
+
"""Atomically lease, load, and reverify one retained resource."""
|
|
239
|
+
async with self._read_admission:
|
|
240
|
+
async with self._gate:
|
|
241
|
+
self._require_open()
|
|
242
|
+
artifact_id, part_index = self._parse_uri(uri)
|
|
243
|
+
indexed = self._artifacts.get(artifact_id)
|
|
244
|
+
if indexed is None:
|
|
245
|
+
raise ArtifactNotFoundError("resource not found")
|
|
246
|
+
bundle, artifact = indexed
|
|
247
|
+
if part_index is None:
|
|
248
|
+
if artifact.record.representation == "manifest":
|
|
249
|
+
file_name = artifact.manifest_name
|
|
250
|
+
expected_size = artifact.manifest_size
|
|
251
|
+
expected_hash = artifact.manifest_sha256
|
|
252
|
+
expected_identity = artifact.manifest_identity
|
|
253
|
+
mime_type = _MANIFEST_MIME
|
|
254
|
+
text = True
|
|
255
|
+
else:
|
|
256
|
+
part = artifact.parts[0]
|
|
257
|
+
file_name = part.name
|
|
258
|
+
expected_size = part.size
|
|
259
|
+
expected_hash = part.sha256
|
|
260
|
+
expected_identity = part.identity
|
|
261
|
+
mime_type = artifact.record.mime_type
|
|
262
|
+
text = False
|
|
263
|
+
else:
|
|
264
|
+
if artifact.record.representation != "manifest":
|
|
265
|
+
raise ArtifactNotFoundError("resource not found")
|
|
266
|
+
if part_index < 0 or part_index >= len(artifact.parts):
|
|
267
|
+
raise ArtifactNotFoundError("resource not found")
|
|
268
|
+
part = artifact.parts[part_index]
|
|
269
|
+
file_name = part.name
|
|
270
|
+
expected_size = part.size
|
|
271
|
+
expected_hash = part.sha256
|
|
272
|
+
expected_identity = part.identity
|
|
273
|
+
mime_type = "application/octet-stream"
|
|
274
|
+
text = False
|
|
275
|
+
if file_name is None or expected_identity is None:
|
|
276
|
+
raise ArtifactIntegrityError("resource integrity failure")
|
|
277
|
+
bundle.leases += 1
|
|
278
|
+
load_task = asyncio.create_task(
|
|
279
|
+
asyncio.to_thread(
|
|
280
|
+
self._load_file,
|
|
281
|
+
bundle.directory,
|
|
282
|
+
bundle.directory_identity,
|
|
283
|
+
artifact.directory,
|
|
284
|
+
artifact.directory_identity,
|
|
285
|
+
file_name,
|
|
286
|
+
expected_identity,
|
|
287
|
+
)
|
|
288
|
+
)
|
|
289
|
+
try:
|
|
290
|
+
try:
|
|
291
|
+
data = await asyncio.shield(load_task)
|
|
292
|
+
except asyncio.CancelledError:
|
|
293
|
+
try:
|
|
294
|
+
await load_task
|
|
295
|
+
except Exception:
|
|
296
|
+
pass
|
|
297
|
+
raise
|
|
298
|
+
if len(data) > max(ARTIFACT_RESOURCE_PART_BYTES, _MAX_MANIFEST_BYTES):
|
|
299
|
+
raise ArtifactIntegrityError("resource integrity failure")
|
|
300
|
+
if expected_size is not None and len(data) != expected_size:
|
|
301
|
+
raise ArtifactIntegrityError("resource integrity failure")
|
|
302
|
+
if expected_hash is not None and hashlib.sha256(data).hexdigest() != expected_hash:
|
|
303
|
+
raise ArtifactIntegrityError("resource integrity failure")
|
|
304
|
+
if text:
|
|
305
|
+
try:
|
|
306
|
+
data.decode("utf-8", errors="strict")
|
|
307
|
+
except UnicodeDecodeError as error:
|
|
308
|
+
raise ArtifactIntegrityError("resource integrity failure") from error
|
|
309
|
+
return ArtifactResource(uri, mime_type, data, text)
|
|
310
|
+
finally:
|
|
311
|
+
async with self._gate:
|
|
312
|
+
bundle.leases -= 1
|
|
313
|
+
|
|
314
|
+
def _parse_uri(self, uri: str) -> tuple[str, int | None]:
|
|
315
|
+
prefix = f"{_URI_PREFIX}{self._server_id}/"
|
|
316
|
+
if not uri.startswith(prefix):
|
|
317
|
+
raise ArtifactNotFoundError("resource not found")
|
|
318
|
+
suffix = uri[len(prefix) :]
|
|
319
|
+
fields = suffix.split("/")
|
|
320
|
+
if not fields or not _valid_id(fields[0]):
|
|
321
|
+
raise ArtifactNotFoundError("resource not found")
|
|
322
|
+
if len(fields) == 1:
|
|
323
|
+
return fields[0], None
|
|
324
|
+
if len(fields) == 3 and fields[1] == "part" and fields[2].isascii() and fields[2].isdigit():
|
|
325
|
+
if len(fields[2]) > 2:
|
|
326
|
+
raise ArtifactNotFoundError("resource not found")
|
|
327
|
+
if fields[2] != "0" and fields[2].startswith("0"):
|
|
328
|
+
raise ArtifactNotFoundError("resource not found")
|
|
329
|
+
return fields[0], int(fields[2])
|
|
330
|
+
raise ArtifactNotFoundError("resource not found")
|
|
331
|
+
|
|
332
|
+
def _load_file(
|
|
333
|
+
self,
|
|
334
|
+
bundle_name: str,
|
|
335
|
+
bundle_identity: _Identity,
|
|
336
|
+
artifact_name: str,
|
|
337
|
+
artifact_identity: _Identity,
|
|
338
|
+
file_name: str,
|
|
339
|
+
file_identity: _Identity,
|
|
340
|
+
) -> bytes:
|
|
341
|
+
nofollow = getattr(os, "O_NOFOLLOW", 0)
|
|
342
|
+
bundle_fd = os.open(
|
|
343
|
+
bundle_name,
|
|
344
|
+
os.O_RDONLY | os.O_DIRECTORY | nofollow,
|
|
345
|
+
dir_fd=self._root_fd,
|
|
346
|
+
)
|
|
347
|
+
try:
|
|
348
|
+
if _identity(os.fstat(bundle_fd)) != bundle_identity:
|
|
349
|
+
raise ArtifactIntegrityError("resource integrity failure")
|
|
350
|
+
artifact_fd = os.open(
|
|
351
|
+
artifact_name,
|
|
352
|
+
os.O_RDONLY | os.O_DIRECTORY | nofollow,
|
|
353
|
+
dir_fd=bundle_fd,
|
|
354
|
+
)
|
|
355
|
+
try:
|
|
356
|
+
if _identity(os.fstat(artifact_fd)) != artifact_identity:
|
|
357
|
+
raise ArtifactIntegrityError("resource integrity failure")
|
|
358
|
+
descriptor = os.open(file_name, os.O_RDONLY | nofollow, dir_fd=artifact_fd)
|
|
359
|
+
try:
|
|
360
|
+
observed = os.fstat(descriptor)
|
|
361
|
+
if not stat.S_ISREG(observed.st_mode) or _identity(observed) != file_identity:
|
|
362
|
+
raise ArtifactIntegrityError("resource integrity failure")
|
|
363
|
+
chunks: list[bytes] = []
|
|
364
|
+
remaining = max(ARTIFACT_RESOURCE_PART_BYTES, _MAX_MANIFEST_BYTES) + 1
|
|
365
|
+
while remaining:
|
|
366
|
+
chunk = os.read(descriptor, min(65_536, remaining))
|
|
367
|
+
if not chunk:
|
|
368
|
+
break
|
|
369
|
+
chunks.append(chunk)
|
|
370
|
+
remaining -= len(chunk)
|
|
371
|
+
return b"".join(chunks)
|
|
372
|
+
finally:
|
|
373
|
+
os.close(descriptor)
|
|
374
|
+
finally:
|
|
375
|
+
os.close(artifact_fd)
|
|
376
|
+
except (FileNotFoundError, NotADirectoryError, OSError) as error:
|
|
377
|
+
raise ArtifactIntegrityError("resource integrity failure") from error
|
|
378
|
+
finally:
|
|
379
|
+
os.close(bundle_fd)
|
|
380
|
+
|
|
381
|
+
async def cleanup(self) -> None:
|
|
382
|
+
"""Identity-check, tombstone, and descriptor-relatively remove the owned root."""
|
|
383
|
+
async with self._read_admission:
|
|
384
|
+
async with self._gate:
|
|
385
|
+
if self._cleanup_complete:
|
|
386
|
+
return
|
|
387
|
+
self._closed = True
|
|
388
|
+
try:
|
|
389
|
+
for bundle in tuple(self._active):
|
|
390
|
+
bundle._abort_locked()
|
|
391
|
+
except OSError as error:
|
|
392
|
+
raise ArtifactStoreError("Artifact Store cleanup left residue") from error
|
|
393
|
+
try:
|
|
394
|
+
observed = os.stat(
|
|
395
|
+
self._cleanup_name,
|
|
396
|
+
dir_fd=self._parent_fd,
|
|
397
|
+
follow_symlinks=False,
|
|
398
|
+
)
|
|
399
|
+
except OSError as error:
|
|
400
|
+
raise ArtifactStoreError("Artifact Store root identity unavailable") from error
|
|
401
|
+
if not stat.S_ISDIR(observed.st_mode) or _identity(observed) != self._root_identity:
|
|
402
|
+
raise ArtifactStoreError("Artifact Store root identity mismatch")
|
|
403
|
+
try:
|
|
404
|
+
self._validate_root_layout()
|
|
405
|
+
except OSError as error:
|
|
406
|
+
raise ArtifactStoreError(
|
|
407
|
+
"Artifact Store cleanup found an unexpected entry"
|
|
408
|
+
) from error
|
|
409
|
+
if self._cleanup_name == self._root_name:
|
|
410
|
+
self._cleanup_name = self._rename_to_tombstone()
|
|
411
|
+
try:
|
|
412
|
+
for published_bundle in tuple(self._bundles.values()):
|
|
413
|
+
self._revoke_bundle(published_bundle)
|
|
414
|
+
self._retry_deleting_bundles()
|
|
415
|
+
if os.listdir(self._root_fd):
|
|
416
|
+
raise ArtifactStoreError("Artifact Store cleanup left residue")
|
|
417
|
+
os.rmdir(self._cleanup_name, dir_fd=self._parent_fd)
|
|
418
|
+
try:
|
|
419
|
+
os.stat(
|
|
420
|
+
self._cleanup_name,
|
|
421
|
+
dir_fd=self._parent_fd,
|
|
422
|
+
follow_symlinks=False,
|
|
423
|
+
)
|
|
424
|
+
except FileNotFoundError:
|
|
425
|
+
pass
|
|
426
|
+
else:
|
|
427
|
+
raise ArtifactStoreError("Artifact Store cleanup left root residue")
|
|
428
|
+
except OSError as error:
|
|
429
|
+
raise ArtifactStoreError("Artifact Store cleanup left residue") from error
|
|
430
|
+
os.close(self._root_fd)
|
|
431
|
+
os.close(self._parent_fd)
|
|
432
|
+
self._cleanup_complete = True
|
|
433
|
+
|
|
434
|
+
def _rename_to_tombstone(self) -> str:
|
|
435
|
+
for _ in range(_ID_ATTEMPTS):
|
|
436
|
+
identifier = self._id_factory()
|
|
437
|
+
if not _valid_id(identifier):
|
|
438
|
+
raise ArtifactStoreError("ID factory returned an invalid UUIDv4")
|
|
439
|
+
candidate = f"amkernel-tombstone-{identifier}"
|
|
440
|
+
try:
|
|
441
|
+
os.stat(candidate, dir_fd=self._parent_fd, follow_symlinks=False)
|
|
442
|
+
except FileNotFoundError:
|
|
443
|
+
os.rename(
|
|
444
|
+
self._root_name,
|
|
445
|
+
candidate,
|
|
446
|
+
src_dir_fd=self._parent_fd,
|
|
447
|
+
dst_dir_fd=self._parent_fd,
|
|
448
|
+
)
|
|
449
|
+
renamed = os.stat(candidate, dir_fd=self._parent_fd, follow_symlinks=False)
|
|
450
|
+
if not stat.S_ISDIR(renamed.st_mode) or _identity(renamed) != self._root_identity:
|
|
451
|
+
raise ArtifactStoreError("Artifact Store root identity mismatch") from None
|
|
452
|
+
return candidate
|
|
453
|
+
except OSError as error:
|
|
454
|
+
raise ArtifactStoreError("Artifact Store cleanup failed") from error
|
|
455
|
+
raise ArtifactStoreError("cleanup tombstone collision exhaustion")
|
|
456
|
+
|
|
457
|
+
def _validate_root_layout(self) -> None:
|
|
458
|
+
expected = {bundle.directory for bundle in self._bundles.values()}
|
|
459
|
+
for deletion in self._deleting_bundles.values():
|
|
460
|
+
bundle = deletion.bundle
|
|
461
|
+
expected.add(deletion.progress.tombstones.get("bundle", bundle.directory))
|
|
462
|
+
if set(os.listdir(self._root_fd)) != expected:
|
|
463
|
+
raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
|
|
464
|
+
for bundle in self._bundles.values():
|
|
465
|
+
self._validate_bundle_layout(bundle)
|
|
466
|
+
for deletion in self._deleting_bundles.values():
|
|
467
|
+
bundle = deletion.bundle
|
|
468
|
+
name = deletion.progress.tombstones.get("bundle", bundle.directory)
|
|
469
|
+
observed = os.stat(name, dir_fd=self._root_fd, follow_symlinks=False)
|
|
470
|
+
if (
|
|
471
|
+
not stat.S_ISDIR(observed.st_mode)
|
|
472
|
+
or _identity(observed) != bundle.directory_identity
|
|
473
|
+
):
|
|
474
|
+
raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
|
|
475
|
+
|
|
476
|
+
def _validate_bundle_layout(self, bundle: _Bundle) -> None:
|
|
477
|
+
nofollow = getattr(os, "O_NOFOLLOW", 0)
|
|
478
|
+
try:
|
|
479
|
+
descriptor = os.open(
|
|
480
|
+
bundle.directory,
|
|
481
|
+
os.O_RDONLY | os.O_DIRECTORY | nofollow,
|
|
482
|
+
dir_fd=self._root_fd,
|
|
483
|
+
)
|
|
484
|
+
except OSError as error:
|
|
485
|
+
raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry") from error
|
|
486
|
+
try:
|
|
487
|
+
if _identity(os.fstat(descriptor)) != bundle.directory_identity:
|
|
488
|
+
raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
|
|
489
|
+
expected_directories = {artifact.directory for artifact in bundle.artifacts.values()}
|
|
490
|
+
if set(os.listdir(descriptor)) != expected_directories:
|
|
491
|
+
raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
|
|
492
|
+
for artifact in bundle.artifacts.values():
|
|
493
|
+
artifact_fd = os.open(
|
|
494
|
+
artifact.directory,
|
|
495
|
+
os.O_RDONLY | os.O_DIRECTORY | nofollow,
|
|
496
|
+
dir_fd=descriptor,
|
|
497
|
+
)
|
|
498
|
+
try:
|
|
499
|
+
if _identity(os.fstat(artifact_fd)) != artifact.directory_identity:
|
|
500
|
+
raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
|
|
501
|
+
expected_identities = {part.name: part.identity for part in artifact.parts}
|
|
502
|
+
if artifact.manifest_name is not None:
|
|
503
|
+
assert artifact.manifest_identity is not None
|
|
504
|
+
expected_identities[artifact.manifest_name] = artifact.manifest_identity
|
|
505
|
+
if set(os.listdir(artifact_fd)) != set(expected_identities):
|
|
506
|
+
raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
|
|
507
|
+
for name, expected_identity in expected_identities.items():
|
|
508
|
+
observed = os.stat(name, dir_fd=artifact_fd, follow_symlinks=False)
|
|
509
|
+
if (
|
|
510
|
+
not stat.S_ISREG(observed.st_mode)
|
|
511
|
+
or _identity(observed) != expected_identity
|
|
512
|
+
):
|
|
513
|
+
raise ArtifactStoreError(
|
|
514
|
+
"Artifact Store cleanup found an unexpected entry"
|
|
515
|
+
)
|
|
516
|
+
finally:
|
|
517
|
+
os.close(artifact_fd)
|
|
518
|
+
except OSError as error:
|
|
519
|
+
raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry") from error
|
|
520
|
+
finally:
|
|
521
|
+
os.close(descriptor)
|
|
522
|
+
|
|
523
|
+
def _evict_oldest_unleased(self) -> bool:
|
|
524
|
+
for bundle in tuple(self._bundles.values()):
|
|
525
|
+
if bundle.leases:
|
|
526
|
+
continue
|
|
527
|
+
deletion = self._revoke_bundle(bundle)
|
|
528
|
+
try:
|
|
529
|
+
self._remove_published_bundle(deletion)
|
|
530
|
+
except OSError as error:
|
|
531
|
+
raise ArtifactStoreError("Artifact Store bundle deletion failed") from error
|
|
532
|
+
self._complete_deletion(bundle.identifier, deletion)
|
|
533
|
+
return True
|
|
534
|
+
return False
|
|
535
|
+
|
|
536
|
+
def _retry_deleting_bundles(self) -> None:
|
|
537
|
+
for identifier, deletion in tuple(self._deleting_bundles.items()):
|
|
538
|
+
try:
|
|
539
|
+
self._remove_published_bundle(deletion)
|
|
540
|
+
except OSError as error:
|
|
541
|
+
raise ArtifactStoreError("Artifact Store bundle deletion failed") from error
|
|
542
|
+
self._complete_deletion(identifier, deletion)
|
|
543
|
+
|
|
544
|
+
def _complete_deletion(self, identifier: str, deletion: _DeletingBundle) -> None:
|
|
545
|
+
del self._deleting_bundles[identifier]
|
|
546
|
+
self._retained_charge -= deletion.bundle.charge
|
|
547
|
+
|
|
548
|
+
def _revoke_bundle(self, bundle: _Bundle) -> _DeletingBundle:
|
|
549
|
+
deletion = _DeletingBundle(bundle)
|
|
550
|
+
self._deleting_bundles[bundle.identifier] = deletion
|
|
551
|
+
del self._bundles[bundle.identifier]
|
|
552
|
+
for artifact_id in bundle.artifacts:
|
|
553
|
+
del self._artifacts[artifact_id]
|
|
554
|
+
return deletion
|
|
555
|
+
|
|
556
|
+
def _rename_deleting_entry(
|
|
557
|
+
self,
|
|
558
|
+
parent_fd: int,
|
|
559
|
+
name: str,
|
|
560
|
+
expected_identity: _Identity,
|
|
561
|
+
progress: _DeletionProgress,
|
|
562
|
+
key: str,
|
|
563
|
+
) -> str:
|
|
564
|
+
retained = progress.tombstones.get(key)
|
|
565
|
+
if retained is not None:
|
|
566
|
+
observed = os.stat(retained, dir_fd=parent_fd, follow_symlinks=False)
|
|
567
|
+
if _identity(observed) != expected_identity:
|
|
568
|
+
raise ArtifactStoreError("Artifact Store cleanup found a substituted entry")
|
|
569
|
+
return retained
|
|
570
|
+
for _ in range(_ID_ATTEMPTS):
|
|
571
|
+
candidate = f"delete-{uuid.uuid4().hex}"
|
|
572
|
+
try:
|
|
573
|
+
os.stat(candidate, dir_fd=parent_fd, follow_symlinks=False)
|
|
574
|
+
except FileNotFoundError:
|
|
575
|
+
os.rename(name, candidate, src_dir_fd=parent_fd, dst_dir_fd=parent_fd)
|
|
576
|
+
progress.tombstones[key] = candidate
|
|
577
|
+
observed = os.stat(candidate, dir_fd=parent_fd, follow_symlinks=False)
|
|
578
|
+
if _identity(observed) != expected_identity:
|
|
579
|
+
raise ArtifactStoreError(
|
|
580
|
+
"Artifact Store cleanup found a substituted entry"
|
|
581
|
+
) from None
|
|
582
|
+
return candidate
|
|
583
|
+
raise ArtifactStoreError("Artifact Store cleanup tombstone collision exhaustion")
|
|
584
|
+
|
|
585
|
+
def _unlink_deleting_file(
|
|
586
|
+
self,
|
|
587
|
+
parent_fd: int,
|
|
588
|
+
name: str,
|
|
589
|
+
expected_identity: _Identity,
|
|
590
|
+
progress: _DeletionProgress,
|
|
591
|
+
key: str,
|
|
592
|
+
) -> None:
|
|
593
|
+
if key in progress.completed:
|
|
594
|
+
return
|
|
595
|
+
current = progress.tombstones.get(key, name)
|
|
596
|
+
descriptor = os.open(
|
|
597
|
+
current,
|
|
598
|
+
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0),
|
|
599
|
+
dir_fd=parent_fd,
|
|
600
|
+
)
|
|
601
|
+
try:
|
|
602
|
+
observed = os.fstat(descriptor)
|
|
603
|
+
if not stat.S_ISREG(observed.st_mode) or _identity(observed) != expected_identity:
|
|
604
|
+
raise ArtifactStoreError("Artifact Store cleanup found a substituted entry")
|
|
605
|
+
tombstone = self._rename_deleting_entry(
|
|
606
|
+
parent_fd, name, expected_identity, progress, key
|
|
607
|
+
)
|
|
608
|
+
os.unlink(tombstone, dir_fd=parent_fd)
|
|
609
|
+
try:
|
|
610
|
+
os.stat(tombstone, dir_fd=parent_fd, follow_symlinks=False)
|
|
611
|
+
except FileNotFoundError:
|
|
612
|
+
progress.tombstones.pop(key, None)
|
|
613
|
+
progress.completed.add(key)
|
|
614
|
+
else:
|
|
615
|
+
raise ArtifactStoreError("Artifact Store cleanup left file residue")
|
|
616
|
+
finally:
|
|
617
|
+
os.close(descriptor)
|
|
618
|
+
|
|
619
|
+
def _remove_deleting_directory(
|
|
620
|
+
self,
|
|
621
|
+
parent_fd: int,
|
|
622
|
+
name: str,
|
|
623
|
+
expected_identity: _Identity,
|
|
624
|
+
progress: _DeletionProgress,
|
|
625
|
+
key: str,
|
|
626
|
+
) -> None:
|
|
627
|
+
if key in progress.completed:
|
|
628
|
+
return
|
|
629
|
+
current = progress.tombstones.get(key, name)
|
|
630
|
+
descriptor = os.open(
|
|
631
|
+
current,
|
|
632
|
+
os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0),
|
|
633
|
+
dir_fd=parent_fd,
|
|
634
|
+
)
|
|
635
|
+
try:
|
|
636
|
+
observed = os.fstat(descriptor)
|
|
637
|
+
if not stat.S_ISDIR(observed.st_mode) or _identity(observed) != expected_identity:
|
|
638
|
+
raise ArtifactStoreError("Artifact Store cleanup found a substituted entry")
|
|
639
|
+
if os.listdir(descriptor):
|
|
640
|
+
raise ArtifactStoreError("Artifact Store cleanup left directory residue")
|
|
641
|
+
tombstone = self._rename_deleting_entry(
|
|
642
|
+
parent_fd, name, expected_identity, progress, key
|
|
643
|
+
)
|
|
644
|
+
os.rmdir(tombstone, dir_fd=parent_fd)
|
|
645
|
+
try:
|
|
646
|
+
os.stat(tombstone, dir_fd=parent_fd, follow_symlinks=False)
|
|
647
|
+
except FileNotFoundError:
|
|
648
|
+
progress.tombstones.pop(key, None)
|
|
649
|
+
progress.completed.add(key)
|
|
650
|
+
else:
|
|
651
|
+
raise ArtifactStoreError("Artifact Store cleanup left directory residue")
|
|
652
|
+
finally:
|
|
653
|
+
os.close(descriptor)
|
|
654
|
+
|
|
655
|
+
def _rename_owned_entry(self, parent_fd: int, name: str, expected_identity: _Identity) -> str:
|
|
656
|
+
for _ in range(_ID_ATTEMPTS):
|
|
657
|
+
candidate = f"delete-{uuid.uuid4().hex}"
|
|
658
|
+
try:
|
|
659
|
+
os.stat(candidate, dir_fd=parent_fd, follow_symlinks=False)
|
|
660
|
+
except FileNotFoundError:
|
|
661
|
+
os.rename(name, candidate, src_dir_fd=parent_fd, dst_dir_fd=parent_fd)
|
|
662
|
+
observed = os.stat(candidate, dir_fd=parent_fd, follow_symlinks=False)
|
|
663
|
+
if _identity(observed) != expected_identity:
|
|
664
|
+
raise ArtifactStoreError(
|
|
665
|
+
"Artifact Store cleanup found a substituted entry"
|
|
666
|
+
) from None
|
|
667
|
+
return candidate
|
|
668
|
+
raise ArtifactStoreError("Artifact Store cleanup tombstone collision exhaustion")
|
|
669
|
+
|
|
670
|
+
def _unlink_owned_file(self, parent_fd: int, name: str, expected_identity: _Identity) -> None:
|
|
671
|
+
descriptor = os.open(
|
|
672
|
+
name,
|
|
673
|
+
os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0),
|
|
674
|
+
dir_fd=parent_fd,
|
|
675
|
+
)
|
|
676
|
+
try:
|
|
677
|
+
observed = os.fstat(descriptor)
|
|
678
|
+
if not stat.S_ISREG(observed.st_mode) or _identity(observed) != expected_identity:
|
|
679
|
+
raise ArtifactStoreError("Artifact Store cleanup found a substituted entry")
|
|
680
|
+
tombstone = self._rename_owned_entry(parent_fd, name, expected_identity)
|
|
681
|
+
os.unlink(tombstone, dir_fd=parent_fd)
|
|
682
|
+
try:
|
|
683
|
+
os.stat(tombstone, dir_fd=parent_fd, follow_symlinks=False)
|
|
684
|
+
except FileNotFoundError:
|
|
685
|
+
pass
|
|
686
|
+
else:
|
|
687
|
+
raise ArtifactStoreError("Artifact Store cleanup left file residue")
|
|
688
|
+
finally:
|
|
689
|
+
os.close(descriptor)
|
|
690
|
+
|
|
691
|
+
def _remove_owned_directory(
|
|
692
|
+
self,
|
|
693
|
+
parent_fd: int,
|
|
694
|
+
name: str,
|
|
695
|
+
expected_identity: _Identity,
|
|
696
|
+
descriptor: int,
|
|
697
|
+
) -> None:
|
|
698
|
+
observed = os.fstat(descriptor)
|
|
699
|
+
if not stat.S_ISDIR(observed.st_mode) or _identity(observed) != expected_identity:
|
|
700
|
+
raise ArtifactStoreError("Artifact Store cleanup found a substituted entry")
|
|
701
|
+
tombstone = self._rename_owned_entry(parent_fd, name, expected_identity)
|
|
702
|
+
os.rmdir(tombstone, dir_fd=parent_fd)
|
|
703
|
+
try:
|
|
704
|
+
os.stat(tombstone, dir_fd=parent_fd, follow_symlinks=False)
|
|
705
|
+
except FileNotFoundError:
|
|
706
|
+
pass
|
|
707
|
+
else:
|
|
708
|
+
raise ArtifactStoreError("Artifact Store cleanup left directory residue")
|
|
709
|
+
|
|
710
|
+
def _remove_published_bundle(self, deletion: _DeletingBundle) -> None:
|
|
711
|
+
bundle = deletion.bundle
|
|
712
|
+
progress = deletion.progress
|
|
713
|
+
if "bundle" in progress.completed:
|
|
714
|
+
return
|
|
715
|
+
if "bundle" in progress.tombstones:
|
|
716
|
+
self._remove_deleting_directory(
|
|
717
|
+
self._root_fd,
|
|
718
|
+
bundle.directory,
|
|
719
|
+
bundle.directory_identity,
|
|
720
|
+
progress,
|
|
721
|
+
"bundle",
|
|
722
|
+
)
|
|
723
|
+
return
|
|
724
|
+
nofollow = getattr(os, "O_NOFOLLOW", 0)
|
|
725
|
+
bundle_fd = os.open(
|
|
726
|
+
bundle.directory,
|
|
727
|
+
os.O_RDONLY | os.O_DIRECTORY | nofollow,
|
|
728
|
+
dir_fd=self._root_fd,
|
|
729
|
+
)
|
|
730
|
+
try:
|
|
731
|
+
if _identity(os.fstat(bundle_fd)) != bundle.directory_identity:
|
|
732
|
+
raise ArtifactStoreError("Artifact Store cleanup found a substituted entry")
|
|
733
|
+
for artifact_id, artifact in bundle.artifacts.items():
|
|
734
|
+
directory_key = f"artifact:{artifact_id}"
|
|
735
|
+
if directory_key in progress.completed:
|
|
736
|
+
continue
|
|
737
|
+
if directory_key not in progress.tombstones:
|
|
738
|
+
artifact_fd = os.open(
|
|
739
|
+
artifact.directory,
|
|
740
|
+
os.O_RDONLY | os.O_DIRECTORY | nofollow,
|
|
741
|
+
dir_fd=bundle_fd,
|
|
742
|
+
)
|
|
743
|
+
try:
|
|
744
|
+
if _identity(os.fstat(artifact_fd)) != artifact.directory_identity:
|
|
745
|
+
raise ArtifactStoreError(
|
|
746
|
+
"Artifact Store cleanup found a substituted entry"
|
|
747
|
+
)
|
|
748
|
+
expected_files = {part.name: part.identity for part in artifact.parts}
|
|
749
|
+
if artifact.manifest_name is not None:
|
|
750
|
+
assert artifact.manifest_identity is not None
|
|
751
|
+
expected_files[artifact.manifest_name] = artifact.manifest_identity
|
|
752
|
+
for file_name, file_identity in expected_files.items():
|
|
753
|
+
self._unlink_deleting_file(
|
|
754
|
+
artifact_fd,
|
|
755
|
+
file_name,
|
|
756
|
+
file_identity,
|
|
757
|
+
progress,
|
|
758
|
+
f"{directory_key}/file:{file_name}",
|
|
759
|
+
)
|
|
760
|
+
finally:
|
|
761
|
+
os.close(artifact_fd)
|
|
762
|
+
self._remove_deleting_directory(
|
|
763
|
+
bundle_fd,
|
|
764
|
+
artifact.directory,
|
|
765
|
+
artifact.directory_identity,
|
|
766
|
+
progress,
|
|
767
|
+
directory_key,
|
|
768
|
+
)
|
|
769
|
+
finally:
|
|
770
|
+
os.close(bundle_fd)
|
|
771
|
+
self._remove_deleting_directory(
|
|
772
|
+
self._root_fd,
|
|
773
|
+
bundle.directory,
|
|
774
|
+
bundle.directory_identity,
|
|
775
|
+
progress,
|
|
776
|
+
"bundle",
|
|
777
|
+
)
|
|
778
|
+
|
|
779
|
+
def _require_open(self) -> None:
|
|
780
|
+
if self._closed:
|
|
781
|
+
raise ArtifactStoreError("Artifact Store is closed")
|
|
782
|
+
|
|
783
|
+
|
|
784
|
+
class ArtifactBundle:
|
|
785
|
+
"""One reserved execution transaction; publish or abort exactly once."""
|
|
786
|
+
|
|
787
|
+
def __init__(
|
|
788
|
+
self,
|
|
789
|
+
store: ArtifactStore,
|
|
790
|
+
identifier: str,
|
|
791
|
+
stage_name: str,
|
|
792
|
+
stage_identity: _Identity,
|
|
793
|
+
) -> None:
|
|
794
|
+
self._store = store
|
|
795
|
+
self._identifier = identifier
|
|
796
|
+
self._stage_name = stage_name
|
|
797
|
+
self._stage_identity = stage_identity
|
|
798
|
+
self._artifact_ids: set[str] = set()
|
|
799
|
+
self._writers: set[ArtifactWriter] = set()
|
|
800
|
+
self._artifacts: list[_Artifact] = []
|
|
801
|
+
self._parts = 0
|
|
802
|
+
self._charge = 0
|
|
803
|
+
self._settled = False
|
|
804
|
+
self._deletion_progress = _DeletionProgress()
|
|
805
|
+
|
|
806
|
+
@property
|
|
807
|
+
def settled(self) -> bool:
|
|
808
|
+
return self._settled
|
|
809
|
+
|
|
810
|
+
def open_artifact(
|
|
811
|
+
self, name: str, declared_mime_type: str, *, safe_mime_type: str | None = None
|
|
812
|
+
) -> ArtifactWriter:
|
|
813
|
+
"""Open one incremental logical artifact with optional trusted serving MIME."""
|
|
814
|
+
if self._settled:
|
|
815
|
+
raise ArtifactStoreError("bundle is settled")
|
|
816
|
+
if len(self._writers) + len(self._artifacts) >= MAX_STORED_ITEMS:
|
|
817
|
+
raise ArtifactStoreError("execution artifact item limit exceeded")
|
|
818
|
+
validate_logical_name(name)
|
|
819
|
+
validate_mime_type(declared_mime_type)
|
|
820
|
+
if safe_mime_type is not None:
|
|
821
|
+
validate_mime_type(safe_mime_type)
|
|
822
|
+
artifact_id, directory, directory_identity = self._create_artifact_directory()
|
|
823
|
+
self._artifact_ids.add(artifact_id)
|
|
824
|
+
writer = ArtifactWriter(
|
|
825
|
+
self,
|
|
826
|
+
artifact_id,
|
|
827
|
+
directory,
|
|
828
|
+
directory_identity,
|
|
829
|
+
name,
|
|
830
|
+
declared_mime_type,
|
|
831
|
+
safe_mime_type,
|
|
832
|
+
)
|
|
833
|
+
self._writers.add(writer)
|
|
834
|
+
return writer
|
|
835
|
+
|
|
836
|
+
def _create_artifact_directory(self) -> tuple[str, str, _Identity]:
|
|
837
|
+
stage_fd = os.open(
|
|
838
|
+
self._stage_name,
|
|
839
|
+
os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0),
|
|
840
|
+
dir_fd=self._store._root_fd,
|
|
841
|
+
)
|
|
842
|
+
try:
|
|
843
|
+
if _identity(os.fstat(stage_fd)) != self._stage_identity:
|
|
844
|
+
raise ArtifactStoreError("bundle staging identity mismatch")
|
|
845
|
+
for _ in range(_ID_ATTEMPTS):
|
|
846
|
+
identifier = self._store._id_factory()
|
|
847
|
+
if not _valid_id(identifier):
|
|
848
|
+
raise ArtifactStoreError("ID factory returned an invalid UUIDv4")
|
|
849
|
+
name = f"artifact-{identifier}"
|
|
850
|
+
if (
|
|
851
|
+
identifier in self._store._artifacts
|
|
852
|
+
or identifier in self._store._reserved_artifact_ids
|
|
853
|
+
):
|
|
854
|
+
continue
|
|
855
|
+
try:
|
|
856
|
+
os.mkdir(name, mode=0o700, dir_fd=stage_fd)
|
|
857
|
+
except FileExistsError:
|
|
858
|
+
continue
|
|
859
|
+
os.chmod(name, 0o700, dir_fd=stage_fd, follow_symlinks=False)
|
|
860
|
+
observed = os.stat(name, dir_fd=stage_fd, follow_symlinks=False)
|
|
861
|
+
self._store._reserved_artifact_ids.add(identifier)
|
|
862
|
+
return identifier, name, _identity(observed)
|
|
863
|
+
finally:
|
|
864
|
+
os.close(stage_fd)
|
|
865
|
+
raise ArtifactStoreError("artifact ID collision exhaustion")
|
|
866
|
+
|
|
867
|
+
async def publish(self) -> tuple[ArtifactRecord, ...]:
|
|
868
|
+
"""Atomically make every finished item visible."""
|
|
869
|
+
async with self._store._gate:
|
|
870
|
+
if self._settled or self not in self._store._active:
|
|
871
|
+
raise ArtifactStoreError("bundle is settled")
|
|
872
|
+
if self._writers:
|
|
873
|
+
raise ArtifactStoreError("bundle has unfinished artifacts")
|
|
874
|
+
artifacts = {
|
|
875
|
+
artifact.record.uri.rsplit("/", 1)[-1]: artifact for artifact in self._artifacts
|
|
876
|
+
}
|
|
877
|
+
if set(artifacts) != self._artifact_ids or any(
|
|
878
|
+
identifier in self._store._artifacts for identifier in artifacts
|
|
879
|
+
):
|
|
880
|
+
raise ArtifactStoreError("artifact ID was already published")
|
|
881
|
+
observed_stage = os.stat(
|
|
882
|
+
self._stage_name,
|
|
883
|
+
dir_fd=self._store._root_fd,
|
|
884
|
+
follow_symlinks=False,
|
|
885
|
+
)
|
|
886
|
+
if _identity(observed_stage) != self._stage_identity:
|
|
887
|
+
raise ArtifactStoreError("bundle staging identity mismatch")
|
|
888
|
+
published_name = f"bundle-{self._identifier}"
|
|
889
|
+
try:
|
|
890
|
+
os.stat(published_name, dir_fd=self._store._root_fd, follow_symlinks=False)
|
|
891
|
+
except FileNotFoundError:
|
|
892
|
+
pass
|
|
893
|
+
else:
|
|
894
|
+
raise ArtifactStoreError("published bundle name collision")
|
|
895
|
+
try:
|
|
896
|
+
os.rename(
|
|
897
|
+
self._stage_name,
|
|
898
|
+
published_name,
|
|
899
|
+
src_dir_fd=self._store._root_fd,
|
|
900
|
+
dst_dir_fd=self._store._root_fd,
|
|
901
|
+
)
|
|
902
|
+
except OSError as error:
|
|
903
|
+
self._abort_locked()
|
|
904
|
+
raise ArtifactStoreError("bundle publication failed") from error
|
|
905
|
+
published = os.stat(
|
|
906
|
+
published_name,
|
|
907
|
+
dir_fd=self._store._root_fd,
|
|
908
|
+
follow_symlinks=False,
|
|
909
|
+
)
|
|
910
|
+
if _identity(published) != self._stage_identity:
|
|
911
|
+
raise ArtifactStoreError("published bundle identity mismatch")
|
|
912
|
+
bundle = _Bundle(
|
|
913
|
+
self._identifier,
|
|
914
|
+
published_name,
|
|
915
|
+
artifacts,
|
|
916
|
+
self._stage_identity,
|
|
917
|
+
self._charge,
|
|
918
|
+
)
|
|
919
|
+
self._store._bundles[self._identifier] = bundle
|
|
920
|
+
for artifact_id, artifact in artifacts.items():
|
|
921
|
+
self._store._artifacts[artifact_id] = (bundle, artifact)
|
|
922
|
+
self._store._reserved_artifact_ids.difference_update(self._artifact_ids)
|
|
923
|
+
self._store._retained_charge += self._charge
|
|
924
|
+
self._store._reserved_charge -= self._store._reservation_bytes
|
|
925
|
+
self._store._active.remove(self)
|
|
926
|
+
self._settled = True
|
|
927
|
+
return tuple(artifact.record for artifact in self._artifacts)
|
|
928
|
+
|
|
929
|
+
async def discard(self) -> None:
|
|
930
|
+
"""Remove this execution bundle whether staged or already published."""
|
|
931
|
+
async with self._store._gate:
|
|
932
|
+
if not self._settled:
|
|
933
|
+
self._abort_locked()
|
|
934
|
+
return
|
|
935
|
+
deletion = self._store._deleting_bundles.get(self._identifier)
|
|
936
|
+
if deletion is None:
|
|
937
|
+
published = self._store._bundles.get(self._identifier)
|
|
938
|
+
if published is None:
|
|
939
|
+
return
|
|
940
|
+
if published.leases:
|
|
941
|
+
raise ArtifactStoreError("execution bundle is in use")
|
|
942
|
+
deletion = self._store._revoke_bundle(published)
|
|
943
|
+
try:
|
|
944
|
+
self._store._remove_published_bundle(deletion)
|
|
945
|
+
except OSError as error:
|
|
946
|
+
raise ArtifactStoreError("Artifact Store bundle deletion failed") from error
|
|
947
|
+
self._store._complete_deletion(self._identifier, deletion)
|
|
948
|
+
|
|
949
|
+
async def abort(self) -> None:
|
|
950
|
+
async with self._store._gate:
|
|
951
|
+
try:
|
|
952
|
+
self._abort_locked()
|
|
953
|
+
except OSError as error:
|
|
954
|
+
raise ArtifactStoreError("Artifact Store bundle deletion failed") from error
|
|
955
|
+
|
|
956
|
+
def _abort_locked(self) -> None:
|
|
957
|
+
if self._settled:
|
|
958
|
+
return
|
|
959
|
+
progress = self._deletion_progress
|
|
960
|
+
current_stage = progress.tombstones.get("stage", self._stage_name)
|
|
961
|
+
try:
|
|
962
|
+
stage_fd = os.open(
|
|
963
|
+
current_stage,
|
|
964
|
+
os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0),
|
|
965
|
+
dir_fd=self._store._root_fd,
|
|
966
|
+
)
|
|
967
|
+
except OSError as error:
|
|
968
|
+
raise ArtifactStoreError("bundle staging identity mismatch") from error
|
|
969
|
+
try:
|
|
970
|
+
if _identity(os.fstat(stage_fd)) != self._stage_identity:
|
|
971
|
+
raise ArtifactStoreError("bundle staging identity mismatch")
|
|
972
|
+
self._remove_staged_contents(stage_fd)
|
|
973
|
+
self._store._remove_deleting_directory(
|
|
974
|
+
self._store._root_fd,
|
|
975
|
+
self._stage_name,
|
|
976
|
+
self._stage_identity,
|
|
977
|
+
progress,
|
|
978
|
+
"stage",
|
|
979
|
+
)
|
|
980
|
+
finally:
|
|
981
|
+
os.close(stage_fd)
|
|
982
|
+
self._store._active.discard(self)
|
|
983
|
+
self._store._reserved_artifact_ids.difference_update(self._artifact_ids)
|
|
984
|
+
self._store._reserved_charge -= self._store._reservation_bytes
|
|
985
|
+
self._settled = True
|
|
986
|
+
|
|
987
|
+
def _remove_staged_contents(self, stage_fd: int) -> None:
|
|
988
|
+
entries: list[tuple[str, _Identity, dict[str, _Identity]]] = []
|
|
989
|
+
for artifact in self._artifacts:
|
|
990
|
+
files = {part.name: part.identity for part in artifact.parts}
|
|
991
|
+
if artifact.manifest_name is not None:
|
|
992
|
+
assert artifact.manifest_identity is not None
|
|
993
|
+
files[artifact.manifest_name] = artifact.manifest_identity
|
|
994
|
+
entries.append((artifact.directory, artifact.directory_identity, files))
|
|
995
|
+
for writer in self._writers:
|
|
996
|
+
entries.append(
|
|
997
|
+
(
|
|
998
|
+
writer._directory,
|
|
999
|
+
writer._directory_identity,
|
|
1000
|
+
{part.name: part.identity for part in writer._parts},
|
|
1001
|
+
)
|
|
1002
|
+
)
|
|
1003
|
+
progress = self._deletion_progress
|
|
1004
|
+
expected_directories = {
|
|
1005
|
+
progress.tombstones.get(key, directory)
|
|
1006
|
+
for directory, _, _ in entries
|
|
1007
|
+
if (key := f"artifact:{directory}") not in progress.completed
|
|
1008
|
+
}
|
|
1009
|
+
if set(os.listdir(stage_fd)) != expected_directories:
|
|
1010
|
+
raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
|
|
1011
|
+
nofollow = getattr(os, "O_NOFOLLOW", 0)
|
|
1012
|
+
for directory, directory_identity, files in entries:
|
|
1013
|
+
directory_key = f"artifact:{directory}"
|
|
1014
|
+
if directory_key in progress.completed:
|
|
1015
|
+
continue
|
|
1016
|
+
current_directory = progress.tombstones.get(directory_key, directory)
|
|
1017
|
+
artifact_fd = os.open(
|
|
1018
|
+
current_directory,
|
|
1019
|
+
os.O_RDONLY | os.O_DIRECTORY | nofollow,
|
|
1020
|
+
dir_fd=stage_fd,
|
|
1021
|
+
)
|
|
1022
|
+
try:
|
|
1023
|
+
if _identity(os.fstat(artifact_fd)) != directory_identity:
|
|
1024
|
+
raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
|
|
1025
|
+
expected_files = {
|
|
1026
|
+
progress.tombstones.get(key, name)
|
|
1027
|
+
for name in files
|
|
1028
|
+
if (key := f"{directory_key}/file:{name}") not in progress.completed
|
|
1029
|
+
}
|
|
1030
|
+
if set(os.listdir(artifact_fd)) != expected_files:
|
|
1031
|
+
raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
|
|
1032
|
+
for name, file_identity in files.items():
|
|
1033
|
+
self._store._unlink_deleting_file(
|
|
1034
|
+
artifact_fd,
|
|
1035
|
+
name,
|
|
1036
|
+
file_identity,
|
|
1037
|
+
progress,
|
|
1038
|
+
f"{directory_key}/file:{name}",
|
|
1039
|
+
)
|
|
1040
|
+
self._store._remove_deleting_directory(
|
|
1041
|
+
stage_fd,
|
|
1042
|
+
directory,
|
|
1043
|
+
directory_identity,
|
|
1044
|
+
progress,
|
|
1045
|
+
directory_key,
|
|
1046
|
+
)
|
|
1047
|
+
finally:
|
|
1048
|
+
os.close(artifact_fd)
|
|
1049
|
+
|
|
1050
|
+
def _discard(self, writer: ArtifactWriter) -> None:
|
|
1051
|
+
if writer not in self._writers or self._settled:
|
|
1052
|
+
raise ArtifactStoreError("artifact writer is not active")
|
|
1053
|
+
nofollow = getattr(os, "O_NOFOLLOW", 0)
|
|
1054
|
+
progress = self._deletion_progress
|
|
1055
|
+
directory_key = f"artifact:{writer._directory}"
|
|
1056
|
+
try:
|
|
1057
|
+
current_stage = progress.tombstones.get("stage", self._stage_name)
|
|
1058
|
+
stage_fd = os.open(
|
|
1059
|
+
current_stage,
|
|
1060
|
+
os.O_RDONLY | os.O_DIRECTORY | nofollow,
|
|
1061
|
+
dir_fd=self._store._root_fd,
|
|
1062
|
+
)
|
|
1063
|
+
try:
|
|
1064
|
+
if _identity(os.fstat(stage_fd)) != self._stage_identity:
|
|
1065
|
+
raise ArtifactStoreError("bundle staging identity mismatch")
|
|
1066
|
+
current_directory = progress.tombstones.get(directory_key, writer._directory)
|
|
1067
|
+
artifact_fd = os.open(
|
|
1068
|
+
current_directory,
|
|
1069
|
+
os.O_RDONLY | os.O_DIRECTORY | nofollow,
|
|
1070
|
+
dir_fd=stage_fd,
|
|
1071
|
+
)
|
|
1072
|
+
try:
|
|
1073
|
+
if _identity(os.fstat(artifact_fd)) != writer._directory_identity:
|
|
1074
|
+
raise ArtifactStoreError("artifact staging identity mismatch")
|
|
1075
|
+
files = {part.name: part.identity for part in writer._parts}
|
|
1076
|
+
expected_files = {
|
|
1077
|
+
progress.tombstones.get(key, name)
|
|
1078
|
+
for name in files
|
|
1079
|
+
if (key := f"{directory_key}/file:{name}") not in progress.completed
|
|
1080
|
+
}
|
|
1081
|
+
if set(os.listdir(artifact_fd)) != expected_files:
|
|
1082
|
+
raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
|
|
1083
|
+
for name, identity in files.items():
|
|
1084
|
+
self._store._unlink_deleting_file(
|
|
1085
|
+
artifact_fd,
|
|
1086
|
+
name,
|
|
1087
|
+
identity,
|
|
1088
|
+
progress,
|
|
1089
|
+
f"{directory_key}/file:{name}",
|
|
1090
|
+
)
|
|
1091
|
+
self._store._remove_deleting_directory(
|
|
1092
|
+
stage_fd,
|
|
1093
|
+
writer._directory,
|
|
1094
|
+
writer._directory_identity,
|
|
1095
|
+
progress,
|
|
1096
|
+
directory_key,
|
|
1097
|
+
)
|
|
1098
|
+
finally:
|
|
1099
|
+
os.close(artifact_fd)
|
|
1100
|
+
finally:
|
|
1101
|
+
os.close(stage_fd)
|
|
1102
|
+
except OSError as error:
|
|
1103
|
+
raise ArtifactStoreError("Artifact Store bundle deletion failed") from error
|
|
1104
|
+
self._writers.remove(writer)
|
|
1105
|
+
self._artifact_ids.remove(writer._artifact_id)
|
|
1106
|
+
self._store._reserved_artifact_ids.remove(writer._artifact_id)
|
|
1107
|
+
|
|
1108
|
+
def _validate_finished(self, writer: ArtifactWriter, part_count: int, charge: int) -> int:
|
|
1109
|
+
if writer not in self._writers or self._settled:
|
|
1110
|
+
raise ArtifactStoreError("artifact writer is not active")
|
|
1111
|
+
new_parts = self._parts + part_count
|
|
1112
|
+
if new_parts > MAX_EXECUTION_PARTS:
|
|
1113
|
+
raise ArtifactStoreError("execution artifact part limit exceeded")
|
|
1114
|
+
if self._charge + charge > self._store._reservation_bytes:
|
|
1115
|
+
raise ArtifactStoreError("execution artifact reservation exceeded")
|
|
1116
|
+
return new_parts
|
|
1117
|
+
|
|
1118
|
+
def _finished(self, writer: ArtifactWriter, artifact: _Artifact, charge: int) -> None:
|
|
1119
|
+
new_parts = self._validate_finished(writer, len(artifact.parts), charge)
|
|
1120
|
+
self._writers.remove(writer)
|
|
1121
|
+
self._artifacts.append(artifact)
|
|
1122
|
+
self._parts = new_parts
|
|
1123
|
+
self._charge += charge
|
|
1124
|
+
|
|
1125
|
+
|
|
1126
|
+
class ArtifactWriter:
|
|
1127
|
+
"""Incrementally hash and split one logical artifact into immutable parts."""
|
|
1128
|
+
|
|
1129
|
+
def __init__(
|
|
1130
|
+
self,
|
|
1131
|
+
bundle: ArtifactBundle,
|
|
1132
|
+
artifact_id: str,
|
|
1133
|
+
directory: str,
|
|
1134
|
+
directory_identity: _Identity,
|
|
1135
|
+
name: str,
|
|
1136
|
+
declared_mime_type: str,
|
|
1137
|
+
safe_mime_type: str | None,
|
|
1138
|
+
) -> None:
|
|
1139
|
+
self._bundle = bundle
|
|
1140
|
+
self._artifact_id = artifact_id
|
|
1141
|
+
self._directory = directory
|
|
1142
|
+
self._directory_identity = directory_identity
|
|
1143
|
+
self._name = name
|
|
1144
|
+
self._declared_mime_type = declared_mime_type
|
|
1145
|
+
self._safe_mime_type = safe_mime_type
|
|
1146
|
+
self._hash = hashlib.sha256()
|
|
1147
|
+
self._size = 0
|
|
1148
|
+
self._parts: list[_Part] = []
|
|
1149
|
+
self._buffer: bytearray | None = None
|
|
1150
|
+
self._buffer_size = 0
|
|
1151
|
+
self._finished = False
|
|
1152
|
+
|
|
1153
|
+
def write(self, data: bytes | bytearray | memoryview) -> None:
|
|
1154
|
+
if self._finished:
|
|
1155
|
+
raise ArtifactStoreError("artifact writer is finished")
|
|
1156
|
+
view = memoryview(data)
|
|
1157
|
+
if self._size + len(view) > MAX_ARTIFACT_BYTES:
|
|
1158
|
+
raise ArtifactStoreError("artifact byte limit exceeded")
|
|
1159
|
+
self._hash.update(view)
|
|
1160
|
+
self._size += len(view)
|
|
1161
|
+
while view:
|
|
1162
|
+
if self._buffer is None:
|
|
1163
|
+
self._buffer = bytearray(ARTIFACT_RESOURCE_PART_BYTES)
|
|
1164
|
+
available = ARTIFACT_RESOURCE_PART_BYTES - self._buffer_size
|
|
1165
|
+
take = min(len(view), available)
|
|
1166
|
+
self._buffer[self._buffer_size : self._buffer_size + take] = view[:take]
|
|
1167
|
+
self._buffer_size += take
|
|
1168
|
+
view = view[take:]
|
|
1169
|
+
if self._buffer_size == ARTIFACT_RESOURCE_PART_BYTES:
|
|
1170
|
+
self._write_part()
|
|
1171
|
+
|
|
1172
|
+
def discard(self) -> None:
|
|
1173
|
+
"""Remove one unfinished logical artifact from its active bundle."""
|
|
1174
|
+
if self._finished:
|
|
1175
|
+
raise ArtifactStoreError("artifact writer is finished")
|
|
1176
|
+
try:
|
|
1177
|
+
self._bundle._discard(self)
|
|
1178
|
+
except OSError as error:
|
|
1179
|
+
raise ArtifactStoreError("artifact discard failed") from error
|
|
1180
|
+
self._buffer = None
|
|
1181
|
+
self._buffer_size = 0
|
|
1182
|
+
self._finished = True
|
|
1183
|
+
|
|
1184
|
+
def _set_safe_mime_type(self, mime_type: str) -> None:
|
|
1185
|
+
"""Promote serving MIME after Host content validation."""
|
|
1186
|
+
if self._finished:
|
|
1187
|
+
raise ArtifactStoreError("artifact writer is finished")
|
|
1188
|
+
validate_mime_type(mime_type)
|
|
1189
|
+
self._safe_mime_type = mime_type
|
|
1190
|
+
|
|
1191
|
+
def finish(self) -> ArtifactRecord:
|
|
1192
|
+
if self._finished:
|
|
1193
|
+
raise ArtifactStoreError("artifact writer is finished")
|
|
1194
|
+
if self._buffer_size or not self._parts:
|
|
1195
|
+
self._write_part()
|
|
1196
|
+
if len(self._parts) > MAX_ARTIFACT_PARTS:
|
|
1197
|
+
raise ArtifactStoreError("artifact part limit exceeded")
|
|
1198
|
+
charge = self._size + _ITEM_CHARGE + len(self._parts) * _PART_CHARGE
|
|
1199
|
+
self._bundle._validate_finished(self, len(self._parts), charge)
|
|
1200
|
+
digest = self._hash.hexdigest()
|
|
1201
|
+
uri = f"{_URI_PREFIX}{self._bundle._store.server_id}/{self._artifact_id}"
|
|
1202
|
+
representation: Literal["raw", "manifest"] = (
|
|
1203
|
+
"raw" if self._size <= ARTIFACT_RESOURCE_PART_BYTES else "manifest"
|
|
1204
|
+
)
|
|
1205
|
+
mime_type = (
|
|
1206
|
+
self._safe_mime_type or _safe_mime(self._declared_mime_type)
|
|
1207
|
+
if representation == "raw"
|
|
1208
|
+
else _MANIFEST_MIME
|
|
1209
|
+
)
|
|
1210
|
+
manifest_name: str | None = None
|
|
1211
|
+
if representation == "manifest":
|
|
1212
|
+
manifest = {
|
|
1213
|
+
"version": 1,
|
|
1214
|
+
"name": self._name,
|
|
1215
|
+
"declaredMimeType": self._declared_mime_type,
|
|
1216
|
+
"size": self._size,
|
|
1217
|
+
"sha256": digest,
|
|
1218
|
+
"parts": [
|
|
1219
|
+
{
|
|
1220
|
+
"uri": f"{uri}/part/{index}",
|
|
1221
|
+
"offset": part.offset,
|
|
1222
|
+
"size": part.size,
|
|
1223
|
+
"sha256": part.sha256,
|
|
1224
|
+
}
|
|
1225
|
+
for index, part in enumerate(self._parts)
|
|
1226
|
+
],
|
|
1227
|
+
}
|
|
1228
|
+
encoded = json.dumps(
|
|
1229
|
+
manifest,
|
|
1230
|
+
ensure_ascii=False,
|
|
1231
|
+
allow_nan=False,
|
|
1232
|
+
separators=(",", ":"),
|
|
1233
|
+
).encode("utf-8")
|
|
1234
|
+
if len(encoded) >= _MAX_MANIFEST_BYTES:
|
|
1235
|
+
raise ArtifactStoreError("artifact manifest exceeds its bound")
|
|
1236
|
+
manifest_name = "manifest.json"
|
|
1237
|
+
manifest_size = len(encoded)
|
|
1238
|
+
manifest_sha256 = hashlib.sha256(encoded).hexdigest()
|
|
1239
|
+
manifest_identity = self._write_atomic_file(manifest_name, encoded)
|
|
1240
|
+
else:
|
|
1241
|
+
manifest_size = None
|
|
1242
|
+
manifest_sha256 = None
|
|
1243
|
+
manifest_identity = None
|
|
1244
|
+
record = ArtifactRecord(
|
|
1245
|
+
uri=uri,
|
|
1246
|
+
name=self._name,
|
|
1247
|
+
declared_mime_type=self._declared_mime_type,
|
|
1248
|
+
mime_type=mime_type,
|
|
1249
|
+
representation=representation,
|
|
1250
|
+
size=self._size,
|
|
1251
|
+
sha256=digest,
|
|
1252
|
+
)
|
|
1253
|
+
artifact = _Artifact(
|
|
1254
|
+
record,
|
|
1255
|
+
self._directory,
|
|
1256
|
+
self._directory_identity,
|
|
1257
|
+
tuple(self._parts),
|
|
1258
|
+
manifest_name,
|
|
1259
|
+
manifest_size,
|
|
1260
|
+
manifest_sha256,
|
|
1261
|
+
manifest_identity,
|
|
1262
|
+
)
|
|
1263
|
+
self._bundle._finished(self, artifact, charge)
|
|
1264
|
+
self._finished = True
|
|
1265
|
+
return record
|
|
1266
|
+
|
|
1267
|
+
def _write_part(self) -> None:
|
|
1268
|
+
if len(self._parts) >= MAX_ARTIFACT_PARTS:
|
|
1269
|
+
raise ArtifactStoreError("artifact part limit exceeded")
|
|
1270
|
+
if self._buffer is None:
|
|
1271
|
+
self._buffer = bytearray(ARTIFACT_RESOURCE_PART_BYTES)
|
|
1272
|
+
data = memoryview(self._buffer)[: self._buffer_size]
|
|
1273
|
+
self._buffer_size = 0
|
|
1274
|
+
index = len(self._parts)
|
|
1275
|
+
name = f"part-{index}"
|
|
1276
|
+
identity = self._write_atomic_file(name, data)
|
|
1277
|
+
self._parts.append(
|
|
1278
|
+
_Part(
|
|
1279
|
+
name=name,
|
|
1280
|
+
offset=index * ARTIFACT_RESOURCE_PART_BYTES,
|
|
1281
|
+
size=len(data),
|
|
1282
|
+
sha256=hashlib.sha256(data).hexdigest(),
|
|
1283
|
+
identity=identity,
|
|
1284
|
+
)
|
|
1285
|
+
)
|
|
1286
|
+
|
|
1287
|
+
def _write_atomic_file(
|
|
1288
|
+
self, final_name: str, data: bytes | bytearray | memoryview
|
|
1289
|
+
) -> _Identity:
|
|
1290
|
+
nofollow = getattr(os, "O_NOFOLLOW", 0)
|
|
1291
|
+
stage_fd = os.open(
|
|
1292
|
+
self._bundle._stage_name,
|
|
1293
|
+
os.O_RDONLY | os.O_DIRECTORY | nofollow,
|
|
1294
|
+
dir_fd=self._bundle._store._root_fd,
|
|
1295
|
+
)
|
|
1296
|
+
try:
|
|
1297
|
+
if _identity(os.fstat(stage_fd)) != self._bundle._stage_identity:
|
|
1298
|
+
raise ArtifactStoreError("bundle staging identity mismatch")
|
|
1299
|
+
artifact_fd = os.open(
|
|
1300
|
+
self._directory,
|
|
1301
|
+
os.O_RDONLY | os.O_DIRECTORY | nofollow,
|
|
1302
|
+
dir_fd=stage_fd,
|
|
1303
|
+
)
|
|
1304
|
+
try:
|
|
1305
|
+
if _identity(os.fstat(artifact_fd)) != self._directory_identity:
|
|
1306
|
+
raise ArtifactStoreError("artifact staging identity mismatch")
|
|
1307
|
+
temporary = f"tmp-{uuid.uuid4().hex}"
|
|
1308
|
+
descriptor = os.open(
|
|
1309
|
+
temporary,
|
|
1310
|
+
os.O_WRONLY | os.O_CREAT | os.O_EXCL | nofollow,
|
|
1311
|
+
0o600,
|
|
1312
|
+
dir_fd=artifact_fd,
|
|
1313
|
+
)
|
|
1314
|
+
try:
|
|
1315
|
+
view = memoryview(data)
|
|
1316
|
+
while view:
|
|
1317
|
+
written = os.write(descriptor, view)
|
|
1318
|
+
if written <= 0:
|
|
1319
|
+
raise ArtifactStoreError("artifact write made no progress")
|
|
1320
|
+
view = view[written:]
|
|
1321
|
+
os.fsync(descriptor)
|
|
1322
|
+
observed = os.fstat(descriptor)
|
|
1323
|
+
if observed.st_size != len(data):
|
|
1324
|
+
raise ArtifactStoreError("artifact write size mismatch")
|
|
1325
|
+
os.fchmod(descriptor, 0o400)
|
|
1326
|
+
file_identity = _identity(observed)
|
|
1327
|
+
except BaseException:
|
|
1328
|
+
try:
|
|
1329
|
+
os.unlink(temporary, dir_fd=artifact_fd)
|
|
1330
|
+
except OSError:
|
|
1331
|
+
pass
|
|
1332
|
+
raise
|
|
1333
|
+
finally:
|
|
1334
|
+
os.close(descriptor)
|
|
1335
|
+
try:
|
|
1336
|
+
try:
|
|
1337
|
+
os.stat(final_name, dir_fd=artifact_fd, follow_symlinks=False)
|
|
1338
|
+
except FileNotFoundError:
|
|
1339
|
+
pass
|
|
1340
|
+
else:
|
|
1341
|
+
raise ArtifactStoreError("artifact file already exists")
|
|
1342
|
+
os.rename(
|
|
1343
|
+
temporary,
|
|
1344
|
+
final_name,
|
|
1345
|
+
src_dir_fd=artifact_fd,
|
|
1346
|
+
dst_dir_fd=artifact_fd,
|
|
1347
|
+
)
|
|
1348
|
+
published = os.stat(final_name, dir_fd=artifact_fd, follow_symlinks=False)
|
|
1349
|
+
if _identity(published) != file_identity:
|
|
1350
|
+
raise ArtifactStoreError("artifact file identity mismatch")
|
|
1351
|
+
return file_identity
|
|
1352
|
+
except BaseException:
|
|
1353
|
+
try:
|
|
1354
|
+
os.unlink(temporary, dir_fd=artifact_fd)
|
|
1355
|
+
except OSError:
|
|
1356
|
+
pass
|
|
1357
|
+
raise
|
|
1358
|
+
finally:
|
|
1359
|
+
os.close(artifact_fd)
|
|
1360
|
+
except OSError as error:
|
|
1361
|
+
raise ArtifactStoreError("artifact write failed") from error
|
|
1362
|
+
finally:
|
|
1363
|
+
os.close(stage_fd)
|