amkernel 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1363 @@
1
+ """Bounded process-scoped storage for immutable execution artifacts."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import hashlib
7
+ import json
8
+ import os
9
+ import stat
10
+ import tempfile
11
+ import uuid
12
+ from collections import OrderedDict
13
+ from collections.abc import Callable
14
+ from dataclasses import dataclass, field
15
+ from pathlib import Path
16
+ from typing import Final, Literal
17
+
18
+ from amkernel.protocol import (
19
+ ARTIFACT_RESOURCE_PART_BYTES,
20
+ ARTIFACT_STORE_QUOTA_BYTES,
21
+ EXECUTION_BUNDLE_RESERVATION_BYTES,
22
+ MAX_ARTIFACT_BYTES,
23
+ MAX_ARTIFACT_PARTS,
24
+ MAX_EXECUTION_PARTS,
25
+ MAX_STORED_ITEMS,
26
+ SAFE_MIME_TYPES,
27
+ validate_logical_name,
28
+ validate_mime_type,
29
+ )
30
+
31
+ _ITEM_CHARGE: Final = 4_096
32
+ _PART_CHARGE: Final = 4_096
33
+ _MANIFEST_MIME: Final = "application/vnd.amkernel.artifact-manifest+json"
34
+ _URI_PREFIX: Final = "amkernel://artifact/"
35
+ _MAX_MANIFEST_BYTES: Final = 65_536
36
+ _ID_ATTEMPTS: Final = 3
37
+
38
+
39
+ class ArtifactStoreError(RuntimeError):
40
+ """Artifact storage, quota, integrity, or cleanup failed."""
41
+
42
+
43
+ class ArtifactNotFoundError(ArtifactStoreError):
44
+ """A resource URI is malformed, foreign, or no longer retained."""
45
+
46
+
47
+ class ArtifactIntegrityError(ArtifactStoreError):
48
+ """Retained bytes no longer match their immutable metadata."""
49
+
50
+
51
+ @dataclass(frozen=True, slots=True)
52
+ class ArtifactRecord:
53
+ """Path-free immutable metadata published to callers."""
54
+
55
+ uri: str
56
+ name: str
57
+ declared_mime_type: str
58
+ mime_type: str
59
+ representation: Literal["raw", "manifest"]
60
+ size: int
61
+ sha256: str
62
+
63
+
64
+ @dataclass(frozen=True, slots=True)
65
+ class ArtifactResource:
66
+ """One bounded, verified resource response."""
67
+
68
+ uri: str
69
+ mime_type: str
70
+ data: bytes
71
+ text: bool
72
+
73
+
74
+ _Identity = tuple[int, int]
75
+
76
+
77
+ def _identity(observed: os.stat_result) -> _Identity:
78
+ return observed.st_dev, observed.st_ino
79
+
80
+
81
+ @dataclass(frozen=True, slots=True)
82
+ class _Part:
83
+ name: str
84
+ offset: int
85
+ size: int
86
+ sha256: str
87
+ identity: _Identity
88
+
89
+
90
+ @dataclass(frozen=True, slots=True)
91
+ class _Artifact:
92
+ record: ArtifactRecord
93
+ directory: str
94
+ directory_identity: _Identity
95
+ parts: tuple[_Part, ...]
96
+ manifest_name: str | None
97
+ manifest_size: int | None
98
+ manifest_sha256: str | None
99
+ manifest_identity: _Identity | None
100
+
101
+
102
+ @dataclass(slots=True)
103
+ class _Bundle:
104
+ identifier: str
105
+ directory: str
106
+ artifacts: dict[str, _Artifact]
107
+ directory_identity: _Identity
108
+ charge: int
109
+ leases: int = 0
110
+
111
+
112
+ @dataclass(slots=True)
113
+ class _DeletionProgress:
114
+ tombstones: dict[str, str] = field(default_factory=dict)
115
+ completed: set[str] = field(default_factory=set)
116
+
117
+
118
+ @dataclass(slots=True)
119
+ class _DeletingBundle:
120
+ bundle: _Bundle
121
+ progress: _DeletionProgress = field(default_factory=_DeletionProgress)
122
+
123
+
124
+ IdFactory = Callable[[], str]
125
+
126
+
127
+ def _new_id() -> str:
128
+ return uuid.uuid4().hex
129
+
130
+
131
+ def _valid_id(value: str) -> bool:
132
+ try:
133
+ parsed = uuid.UUID(hex=value)
134
+ except (ValueError, AttributeError):
135
+ return False
136
+ return parsed.version == 4 and parsed.hex == value
137
+
138
+
139
+ def _safe_mime(declared: str) -> str:
140
+ return declared if declared in SAFE_MIME_TYPES else "application/octet-stream"
141
+
142
+
143
+ class ArtifactStore:
144
+ """Own one private root and serve bounded immutable artifact resources."""
145
+
146
+ def __init__(
147
+ self,
148
+ *,
149
+ temporary_parent: Path | None = None,
150
+ id_factory: IdFactory = _new_id,
151
+ quota_bytes: int = ARTIFACT_STORE_QUOTA_BYTES,
152
+ reservation_bytes: int = EXECUTION_BUNDLE_RESERVATION_BYTES,
153
+ ) -> None:
154
+ if quota_bytes < reservation_bytes:
155
+ raise ValueError("artifact quota must fit one execution reservation")
156
+ if os.name != "posix":
157
+ raise ArtifactStoreError("Artifact Store requires POSIX descriptor operations")
158
+ self._id_factory = id_factory
159
+ self._quota_bytes = quota_bytes
160
+ self._reservation_bytes = reservation_bytes
161
+ self._parent = Path(temporary_parent or tempfile.gettempdir())
162
+ self._parent_fd = os.open(self._parent, os.O_RDONLY | os.O_DIRECTORY)
163
+ self._server_id, self._root_name = self._create_root()
164
+ nofollow = getattr(os, "O_NOFOLLOW", 0)
165
+ self._root_fd = os.open(
166
+ self._root_name,
167
+ os.O_RDONLY | os.O_DIRECTORY | nofollow,
168
+ dir_fd=self._parent_fd,
169
+ )
170
+ root_stat = os.fstat(self._root_fd)
171
+ self._root_identity = _identity(root_stat)
172
+ os.fchmod(self._root_fd, 0o700)
173
+ self._deleting_bundles: dict[str, _DeletingBundle] = {}
174
+ self._bundles: OrderedDict[str, _Bundle] = OrderedDict()
175
+ self._artifacts: dict[str, tuple[_Bundle, _Artifact]] = {}
176
+ self._reserved_artifact_ids: set[str] = set()
177
+ self._active: set[ArtifactBundle] = set()
178
+ self._retained_charge = 0
179
+ self._reserved_charge = 0
180
+ self._gate = asyncio.Lock()
181
+ self._read_admission = asyncio.Semaphore(1)
182
+ self._closed = False
183
+ self._cleanup_complete = False
184
+ self._cleanup_name = self._root_name
185
+
186
+ @property
187
+ def server_id(self) -> str:
188
+ return self._server_id
189
+
190
+ def _create_root(self) -> tuple[str, str]:
191
+ for _ in range(_ID_ATTEMPTS):
192
+ identifier = self._id_factory()
193
+ if not _valid_id(identifier):
194
+ raise ArtifactStoreError("ID factory returned an invalid UUIDv4")
195
+ name = f"amkernel-{identifier}"
196
+ try:
197
+ os.mkdir(name, mode=0o700, dir_fd=self._parent_fd)
198
+ except FileExistsError:
199
+ continue
200
+ return identifier, name
201
+ os.close(self._parent_fd)
202
+ raise ArtifactStoreError("Artifact Store ID collision exhaustion")
203
+
204
+ async def begin_bundle(self) -> ArtifactBundle:
205
+ """Reserve exact call capacity and return one unpublished bundle."""
206
+ async with self._read_admission:
207
+ async with self._gate:
208
+ self._require_open()
209
+ self._retry_deleting_bundles()
210
+ while (
211
+ self._retained_charge + self._reserved_charge + self._reservation_bytes
212
+ > self._quota_bytes
213
+ ):
214
+ if not self._evict_oldest_unleased():
215
+ raise ArtifactStoreError("Artifact Store quota is unavailable")
216
+ identifier, stage_name, stage_identity = self._create_bundle_directory()
217
+ bundle = ArtifactBundle(self, identifier, stage_name, stage_identity)
218
+ self._active.add(bundle)
219
+ self._reserved_charge += self._reservation_bytes
220
+ return bundle
221
+
222
+ def _create_bundle_directory(self) -> tuple[str, str, _Identity]:
223
+ for _ in range(_ID_ATTEMPTS):
224
+ identifier = self._id_factory()
225
+ if not _valid_id(identifier):
226
+ raise ArtifactStoreError("ID factory returned an invalid UUIDv4")
227
+ name = f"stage-{identifier}"
228
+ try:
229
+ os.mkdir(name, mode=0o700, dir_fd=self._root_fd)
230
+ except FileExistsError:
231
+ continue
232
+ os.chmod(name, 0o700, dir_fd=self._root_fd, follow_symlinks=False)
233
+ observed = os.stat(name, dir_fd=self._root_fd, follow_symlinks=False)
234
+ return identifier, name, _identity(observed)
235
+ raise ArtifactStoreError("bundle ID collision exhaustion")
236
+
237
+ async def read(self, uri: str) -> ArtifactResource:
238
+ """Atomically lease, load, and reverify one retained resource."""
239
+ async with self._read_admission:
240
+ async with self._gate:
241
+ self._require_open()
242
+ artifact_id, part_index = self._parse_uri(uri)
243
+ indexed = self._artifacts.get(artifact_id)
244
+ if indexed is None:
245
+ raise ArtifactNotFoundError("resource not found")
246
+ bundle, artifact = indexed
247
+ if part_index is None:
248
+ if artifact.record.representation == "manifest":
249
+ file_name = artifact.manifest_name
250
+ expected_size = artifact.manifest_size
251
+ expected_hash = artifact.manifest_sha256
252
+ expected_identity = artifact.manifest_identity
253
+ mime_type = _MANIFEST_MIME
254
+ text = True
255
+ else:
256
+ part = artifact.parts[0]
257
+ file_name = part.name
258
+ expected_size = part.size
259
+ expected_hash = part.sha256
260
+ expected_identity = part.identity
261
+ mime_type = artifact.record.mime_type
262
+ text = False
263
+ else:
264
+ if artifact.record.representation != "manifest":
265
+ raise ArtifactNotFoundError("resource not found")
266
+ if part_index < 0 or part_index >= len(artifact.parts):
267
+ raise ArtifactNotFoundError("resource not found")
268
+ part = artifact.parts[part_index]
269
+ file_name = part.name
270
+ expected_size = part.size
271
+ expected_hash = part.sha256
272
+ expected_identity = part.identity
273
+ mime_type = "application/octet-stream"
274
+ text = False
275
+ if file_name is None or expected_identity is None:
276
+ raise ArtifactIntegrityError("resource integrity failure")
277
+ bundle.leases += 1
278
+ load_task = asyncio.create_task(
279
+ asyncio.to_thread(
280
+ self._load_file,
281
+ bundle.directory,
282
+ bundle.directory_identity,
283
+ artifact.directory,
284
+ artifact.directory_identity,
285
+ file_name,
286
+ expected_identity,
287
+ )
288
+ )
289
+ try:
290
+ try:
291
+ data = await asyncio.shield(load_task)
292
+ except asyncio.CancelledError:
293
+ try:
294
+ await load_task
295
+ except Exception:
296
+ pass
297
+ raise
298
+ if len(data) > max(ARTIFACT_RESOURCE_PART_BYTES, _MAX_MANIFEST_BYTES):
299
+ raise ArtifactIntegrityError("resource integrity failure")
300
+ if expected_size is not None and len(data) != expected_size:
301
+ raise ArtifactIntegrityError("resource integrity failure")
302
+ if expected_hash is not None and hashlib.sha256(data).hexdigest() != expected_hash:
303
+ raise ArtifactIntegrityError("resource integrity failure")
304
+ if text:
305
+ try:
306
+ data.decode("utf-8", errors="strict")
307
+ except UnicodeDecodeError as error:
308
+ raise ArtifactIntegrityError("resource integrity failure") from error
309
+ return ArtifactResource(uri, mime_type, data, text)
310
+ finally:
311
+ async with self._gate:
312
+ bundle.leases -= 1
313
+
314
+ def _parse_uri(self, uri: str) -> tuple[str, int | None]:
315
+ prefix = f"{_URI_PREFIX}{self._server_id}/"
316
+ if not uri.startswith(prefix):
317
+ raise ArtifactNotFoundError("resource not found")
318
+ suffix = uri[len(prefix) :]
319
+ fields = suffix.split("/")
320
+ if not fields or not _valid_id(fields[0]):
321
+ raise ArtifactNotFoundError("resource not found")
322
+ if len(fields) == 1:
323
+ return fields[0], None
324
+ if len(fields) == 3 and fields[1] == "part" and fields[2].isascii() and fields[2].isdigit():
325
+ if len(fields[2]) > 2:
326
+ raise ArtifactNotFoundError("resource not found")
327
+ if fields[2] != "0" and fields[2].startswith("0"):
328
+ raise ArtifactNotFoundError("resource not found")
329
+ return fields[0], int(fields[2])
330
+ raise ArtifactNotFoundError("resource not found")
331
+
332
+ def _load_file(
333
+ self,
334
+ bundle_name: str,
335
+ bundle_identity: _Identity,
336
+ artifact_name: str,
337
+ artifact_identity: _Identity,
338
+ file_name: str,
339
+ file_identity: _Identity,
340
+ ) -> bytes:
341
+ nofollow = getattr(os, "O_NOFOLLOW", 0)
342
+ bundle_fd = os.open(
343
+ bundle_name,
344
+ os.O_RDONLY | os.O_DIRECTORY | nofollow,
345
+ dir_fd=self._root_fd,
346
+ )
347
+ try:
348
+ if _identity(os.fstat(bundle_fd)) != bundle_identity:
349
+ raise ArtifactIntegrityError("resource integrity failure")
350
+ artifact_fd = os.open(
351
+ artifact_name,
352
+ os.O_RDONLY | os.O_DIRECTORY | nofollow,
353
+ dir_fd=bundle_fd,
354
+ )
355
+ try:
356
+ if _identity(os.fstat(artifact_fd)) != artifact_identity:
357
+ raise ArtifactIntegrityError("resource integrity failure")
358
+ descriptor = os.open(file_name, os.O_RDONLY | nofollow, dir_fd=artifact_fd)
359
+ try:
360
+ observed = os.fstat(descriptor)
361
+ if not stat.S_ISREG(observed.st_mode) or _identity(observed) != file_identity:
362
+ raise ArtifactIntegrityError("resource integrity failure")
363
+ chunks: list[bytes] = []
364
+ remaining = max(ARTIFACT_RESOURCE_PART_BYTES, _MAX_MANIFEST_BYTES) + 1
365
+ while remaining:
366
+ chunk = os.read(descriptor, min(65_536, remaining))
367
+ if not chunk:
368
+ break
369
+ chunks.append(chunk)
370
+ remaining -= len(chunk)
371
+ return b"".join(chunks)
372
+ finally:
373
+ os.close(descriptor)
374
+ finally:
375
+ os.close(artifact_fd)
376
+ except (FileNotFoundError, NotADirectoryError, OSError) as error:
377
+ raise ArtifactIntegrityError("resource integrity failure") from error
378
+ finally:
379
+ os.close(bundle_fd)
380
+
381
+ async def cleanup(self) -> None:
382
+ """Identity-check, tombstone, and descriptor-relatively remove the owned root."""
383
+ async with self._read_admission:
384
+ async with self._gate:
385
+ if self._cleanup_complete:
386
+ return
387
+ self._closed = True
388
+ try:
389
+ for bundle in tuple(self._active):
390
+ bundle._abort_locked()
391
+ except OSError as error:
392
+ raise ArtifactStoreError("Artifact Store cleanup left residue") from error
393
+ try:
394
+ observed = os.stat(
395
+ self._cleanup_name,
396
+ dir_fd=self._parent_fd,
397
+ follow_symlinks=False,
398
+ )
399
+ except OSError as error:
400
+ raise ArtifactStoreError("Artifact Store root identity unavailable") from error
401
+ if not stat.S_ISDIR(observed.st_mode) or _identity(observed) != self._root_identity:
402
+ raise ArtifactStoreError("Artifact Store root identity mismatch")
403
+ try:
404
+ self._validate_root_layout()
405
+ except OSError as error:
406
+ raise ArtifactStoreError(
407
+ "Artifact Store cleanup found an unexpected entry"
408
+ ) from error
409
+ if self._cleanup_name == self._root_name:
410
+ self._cleanup_name = self._rename_to_tombstone()
411
+ try:
412
+ for published_bundle in tuple(self._bundles.values()):
413
+ self._revoke_bundle(published_bundle)
414
+ self._retry_deleting_bundles()
415
+ if os.listdir(self._root_fd):
416
+ raise ArtifactStoreError("Artifact Store cleanup left residue")
417
+ os.rmdir(self._cleanup_name, dir_fd=self._parent_fd)
418
+ try:
419
+ os.stat(
420
+ self._cleanup_name,
421
+ dir_fd=self._parent_fd,
422
+ follow_symlinks=False,
423
+ )
424
+ except FileNotFoundError:
425
+ pass
426
+ else:
427
+ raise ArtifactStoreError("Artifact Store cleanup left root residue")
428
+ except OSError as error:
429
+ raise ArtifactStoreError("Artifact Store cleanup left residue") from error
430
+ os.close(self._root_fd)
431
+ os.close(self._parent_fd)
432
+ self._cleanup_complete = True
433
+
434
+ def _rename_to_tombstone(self) -> str:
435
+ for _ in range(_ID_ATTEMPTS):
436
+ identifier = self._id_factory()
437
+ if not _valid_id(identifier):
438
+ raise ArtifactStoreError("ID factory returned an invalid UUIDv4")
439
+ candidate = f"amkernel-tombstone-{identifier}"
440
+ try:
441
+ os.stat(candidate, dir_fd=self._parent_fd, follow_symlinks=False)
442
+ except FileNotFoundError:
443
+ os.rename(
444
+ self._root_name,
445
+ candidate,
446
+ src_dir_fd=self._parent_fd,
447
+ dst_dir_fd=self._parent_fd,
448
+ )
449
+ renamed = os.stat(candidate, dir_fd=self._parent_fd, follow_symlinks=False)
450
+ if not stat.S_ISDIR(renamed.st_mode) or _identity(renamed) != self._root_identity:
451
+ raise ArtifactStoreError("Artifact Store root identity mismatch") from None
452
+ return candidate
453
+ except OSError as error:
454
+ raise ArtifactStoreError("Artifact Store cleanup failed") from error
455
+ raise ArtifactStoreError("cleanup tombstone collision exhaustion")
456
+
457
+ def _validate_root_layout(self) -> None:
458
+ expected = {bundle.directory for bundle in self._bundles.values()}
459
+ for deletion in self._deleting_bundles.values():
460
+ bundle = deletion.bundle
461
+ expected.add(deletion.progress.tombstones.get("bundle", bundle.directory))
462
+ if set(os.listdir(self._root_fd)) != expected:
463
+ raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
464
+ for bundle in self._bundles.values():
465
+ self._validate_bundle_layout(bundle)
466
+ for deletion in self._deleting_bundles.values():
467
+ bundle = deletion.bundle
468
+ name = deletion.progress.tombstones.get("bundle", bundle.directory)
469
+ observed = os.stat(name, dir_fd=self._root_fd, follow_symlinks=False)
470
+ if (
471
+ not stat.S_ISDIR(observed.st_mode)
472
+ or _identity(observed) != bundle.directory_identity
473
+ ):
474
+ raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
475
+
476
+ def _validate_bundle_layout(self, bundle: _Bundle) -> None:
477
+ nofollow = getattr(os, "O_NOFOLLOW", 0)
478
+ try:
479
+ descriptor = os.open(
480
+ bundle.directory,
481
+ os.O_RDONLY | os.O_DIRECTORY | nofollow,
482
+ dir_fd=self._root_fd,
483
+ )
484
+ except OSError as error:
485
+ raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry") from error
486
+ try:
487
+ if _identity(os.fstat(descriptor)) != bundle.directory_identity:
488
+ raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
489
+ expected_directories = {artifact.directory for artifact in bundle.artifacts.values()}
490
+ if set(os.listdir(descriptor)) != expected_directories:
491
+ raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
492
+ for artifact in bundle.artifacts.values():
493
+ artifact_fd = os.open(
494
+ artifact.directory,
495
+ os.O_RDONLY | os.O_DIRECTORY | nofollow,
496
+ dir_fd=descriptor,
497
+ )
498
+ try:
499
+ if _identity(os.fstat(artifact_fd)) != artifact.directory_identity:
500
+ raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
501
+ expected_identities = {part.name: part.identity for part in artifact.parts}
502
+ if artifact.manifest_name is not None:
503
+ assert artifact.manifest_identity is not None
504
+ expected_identities[artifact.manifest_name] = artifact.manifest_identity
505
+ if set(os.listdir(artifact_fd)) != set(expected_identities):
506
+ raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
507
+ for name, expected_identity in expected_identities.items():
508
+ observed = os.stat(name, dir_fd=artifact_fd, follow_symlinks=False)
509
+ if (
510
+ not stat.S_ISREG(observed.st_mode)
511
+ or _identity(observed) != expected_identity
512
+ ):
513
+ raise ArtifactStoreError(
514
+ "Artifact Store cleanup found an unexpected entry"
515
+ )
516
+ finally:
517
+ os.close(artifact_fd)
518
+ except OSError as error:
519
+ raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry") from error
520
+ finally:
521
+ os.close(descriptor)
522
+
523
+ def _evict_oldest_unleased(self) -> bool:
524
+ for bundle in tuple(self._bundles.values()):
525
+ if bundle.leases:
526
+ continue
527
+ deletion = self._revoke_bundle(bundle)
528
+ try:
529
+ self._remove_published_bundle(deletion)
530
+ except OSError as error:
531
+ raise ArtifactStoreError("Artifact Store bundle deletion failed") from error
532
+ self._complete_deletion(bundle.identifier, deletion)
533
+ return True
534
+ return False
535
+
536
+ def _retry_deleting_bundles(self) -> None:
537
+ for identifier, deletion in tuple(self._deleting_bundles.items()):
538
+ try:
539
+ self._remove_published_bundle(deletion)
540
+ except OSError as error:
541
+ raise ArtifactStoreError("Artifact Store bundle deletion failed") from error
542
+ self._complete_deletion(identifier, deletion)
543
+
544
+ def _complete_deletion(self, identifier: str, deletion: _DeletingBundle) -> None:
545
+ del self._deleting_bundles[identifier]
546
+ self._retained_charge -= deletion.bundle.charge
547
+
548
+ def _revoke_bundle(self, bundle: _Bundle) -> _DeletingBundle:
549
+ deletion = _DeletingBundle(bundle)
550
+ self._deleting_bundles[bundle.identifier] = deletion
551
+ del self._bundles[bundle.identifier]
552
+ for artifact_id in bundle.artifacts:
553
+ del self._artifacts[artifact_id]
554
+ return deletion
555
+
556
+ def _rename_deleting_entry(
557
+ self,
558
+ parent_fd: int,
559
+ name: str,
560
+ expected_identity: _Identity,
561
+ progress: _DeletionProgress,
562
+ key: str,
563
+ ) -> str:
564
+ retained = progress.tombstones.get(key)
565
+ if retained is not None:
566
+ observed = os.stat(retained, dir_fd=parent_fd, follow_symlinks=False)
567
+ if _identity(observed) != expected_identity:
568
+ raise ArtifactStoreError("Artifact Store cleanup found a substituted entry")
569
+ return retained
570
+ for _ in range(_ID_ATTEMPTS):
571
+ candidate = f"delete-{uuid.uuid4().hex}"
572
+ try:
573
+ os.stat(candidate, dir_fd=parent_fd, follow_symlinks=False)
574
+ except FileNotFoundError:
575
+ os.rename(name, candidate, src_dir_fd=parent_fd, dst_dir_fd=parent_fd)
576
+ progress.tombstones[key] = candidate
577
+ observed = os.stat(candidate, dir_fd=parent_fd, follow_symlinks=False)
578
+ if _identity(observed) != expected_identity:
579
+ raise ArtifactStoreError(
580
+ "Artifact Store cleanup found a substituted entry"
581
+ ) from None
582
+ return candidate
583
+ raise ArtifactStoreError("Artifact Store cleanup tombstone collision exhaustion")
584
+
585
+ def _unlink_deleting_file(
586
+ self,
587
+ parent_fd: int,
588
+ name: str,
589
+ expected_identity: _Identity,
590
+ progress: _DeletionProgress,
591
+ key: str,
592
+ ) -> None:
593
+ if key in progress.completed:
594
+ return
595
+ current = progress.tombstones.get(key, name)
596
+ descriptor = os.open(
597
+ current,
598
+ os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0),
599
+ dir_fd=parent_fd,
600
+ )
601
+ try:
602
+ observed = os.fstat(descriptor)
603
+ if not stat.S_ISREG(observed.st_mode) or _identity(observed) != expected_identity:
604
+ raise ArtifactStoreError("Artifact Store cleanup found a substituted entry")
605
+ tombstone = self._rename_deleting_entry(
606
+ parent_fd, name, expected_identity, progress, key
607
+ )
608
+ os.unlink(tombstone, dir_fd=parent_fd)
609
+ try:
610
+ os.stat(tombstone, dir_fd=parent_fd, follow_symlinks=False)
611
+ except FileNotFoundError:
612
+ progress.tombstones.pop(key, None)
613
+ progress.completed.add(key)
614
+ else:
615
+ raise ArtifactStoreError("Artifact Store cleanup left file residue")
616
+ finally:
617
+ os.close(descriptor)
618
+
619
+ def _remove_deleting_directory(
620
+ self,
621
+ parent_fd: int,
622
+ name: str,
623
+ expected_identity: _Identity,
624
+ progress: _DeletionProgress,
625
+ key: str,
626
+ ) -> None:
627
+ if key in progress.completed:
628
+ return
629
+ current = progress.tombstones.get(key, name)
630
+ descriptor = os.open(
631
+ current,
632
+ os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0),
633
+ dir_fd=parent_fd,
634
+ )
635
+ try:
636
+ observed = os.fstat(descriptor)
637
+ if not stat.S_ISDIR(observed.st_mode) or _identity(observed) != expected_identity:
638
+ raise ArtifactStoreError("Artifact Store cleanup found a substituted entry")
639
+ if os.listdir(descriptor):
640
+ raise ArtifactStoreError("Artifact Store cleanup left directory residue")
641
+ tombstone = self._rename_deleting_entry(
642
+ parent_fd, name, expected_identity, progress, key
643
+ )
644
+ os.rmdir(tombstone, dir_fd=parent_fd)
645
+ try:
646
+ os.stat(tombstone, dir_fd=parent_fd, follow_symlinks=False)
647
+ except FileNotFoundError:
648
+ progress.tombstones.pop(key, None)
649
+ progress.completed.add(key)
650
+ else:
651
+ raise ArtifactStoreError("Artifact Store cleanup left directory residue")
652
+ finally:
653
+ os.close(descriptor)
654
+
655
+ def _rename_owned_entry(self, parent_fd: int, name: str, expected_identity: _Identity) -> str:
656
+ for _ in range(_ID_ATTEMPTS):
657
+ candidate = f"delete-{uuid.uuid4().hex}"
658
+ try:
659
+ os.stat(candidate, dir_fd=parent_fd, follow_symlinks=False)
660
+ except FileNotFoundError:
661
+ os.rename(name, candidate, src_dir_fd=parent_fd, dst_dir_fd=parent_fd)
662
+ observed = os.stat(candidate, dir_fd=parent_fd, follow_symlinks=False)
663
+ if _identity(observed) != expected_identity:
664
+ raise ArtifactStoreError(
665
+ "Artifact Store cleanup found a substituted entry"
666
+ ) from None
667
+ return candidate
668
+ raise ArtifactStoreError("Artifact Store cleanup tombstone collision exhaustion")
669
+
670
+ def _unlink_owned_file(self, parent_fd: int, name: str, expected_identity: _Identity) -> None:
671
+ descriptor = os.open(
672
+ name,
673
+ os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0),
674
+ dir_fd=parent_fd,
675
+ )
676
+ try:
677
+ observed = os.fstat(descriptor)
678
+ if not stat.S_ISREG(observed.st_mode) or _identity(observed) != expected_identity:
679
+ raise ArtifactStoreError("Artifact Store cleanup found a substituted entry")
680
+ tombstone = self._rename_owned_entry(parent_fd, name, expected_identity)
681
+ os.unlink(tombstone, dir_fd=parent_fd)
682
+ try:
683
+ os.stat(tombstone, dir_fd=parent_fd, follow_symlinks=False)
684
+ except FileNotFoundError:
685
+ pass
686
+ else:
687
+ raise ArtifactStoreError("Artifact Store cleanup left file residue")
688
+ finally:
689
+ os.close(descriptor)
690
+
691
+ def _remove_owned_directory(
692
+ self,
693
+ parent_fd: int,
694
+ name: str,
695
+ expected_identity: _Identity,
696
+ descriptor: int,
697
+ ) -> None:
698
+ observed = os.fstat(descriptor)
699
+ if not stat.S_ISDIR(observed.st_mode) or _identity(observed) != expected_identity:
700
+ raise ArtifactStoreError("Artifact Store cleanup found a substituted entry")
701
+ tombstone = self._rename_owned_entry(parent_fd, name, expected_identity)
702
+ os.rmdir(tombstone, dir_fd=parent_fd)
703
+ try:
704
+ os.stat(tombstone, dir_fd=parent_fd, follow_symlinks=False)
705
+ except FileNotFoundError:
706
+ pass
707
+ else:
708
+ raise ArtifactStoreError("Artifact Store cleanup left directory residue")
709
+
710
+ def _remove_published_bundle(self, deletion: _DeletingBundle) -> None:
711
+ bundle = deletion.bundle
712
+ progress = deletion.progress
713
+ if "bundle" in progress.completed:
714
+ return
715
+ if "bundle" in progress.tombstones:
716
+ self._remove_deleting_directory(
717
+ self._root_fd,
718
+ bundle.directory,
719
+ bundle.directory_identity,
720
+ progress,
721
+ "bundle",
722
+ )
723
+ return
724
+ nofollow = getattr(os, "O_NOFOLLOW", 0)
725
+ bundle_fd = os.open(
726
+ bundle.directory,
727
+ os.O_RDONLY | os.O_DIRECTORY | nofollow,
728
+ dir_fd=self._root_fd,
729
+ )
730
+ try:
731
+ if _identity(os.fstat(bundle_fd)) != bundle.directory_identity:
732
+ raise ArtifactStoreError("Artifact Store cleanup found a substituted entry")
733
+ for artifact_id, artifact in bundle.artifacts.items():
734
+ directory_key = f"artifact:{artifact_id}"
735
+ if directory_key in progress.completed:
736
+ continue
737
+ if directory_key not in progress.tombstones:
738
+ artifact_fd = os.open(
739
+ artifact.directory,
740
+ os.O_RDONLY | os.O_DIRECTORY | nofollow,
741
+ dir_fd=bundle_fd,
742
+ )
743
+ try:
744
+ if _identity(os.fstat(artifact_fd)) != artifact.directory_identity:
745
+ raise ArtifactStoreError(
746
+ "Artifact Store cleanup found a substituted entry"
747
+ )
748
+ expected_files = {part.name: part.identity for part in artifact.parts}
749
+ if artifact.manifest_name is not None:
750
+ assert artifact.manifest_identity is not None
751
+ expected_files[artifact.manifest_name] = artifact.manifest_identity
752
+ for file_name, file_identity in expected_files.items():
753
+ self._unlink_deleting_file(
754
+ artifact_fd,
755
+ file_name,
756
+ file_identity,
757
+ progress,
758
+ f"{directory_key}/file:{file_name}",
759
+ )
760
+ finally:
761
+ os.close(artifact_fd)
762
+ self._remove_deleting_directory(
763
+ bundle_fd,
764
+ artifact.directory,
765
+ artifact.directory_identity,
766
+ progress,
767
+ directory_key,
768
+ )
769
+ finally:
770
+ os.close(bundle_fd)
771
+ self._remove_deleting_directory(
772
+ self._root_fd,
773
+ bundle.directory,
774
+ bundle.directory_identity,
775
+ progress,
776
+ "bundle",
777
+ )
778
+
779
+ def _require_open(self) -> None:
780
+ if self._closed:
781
+ raise ArtifactStoreError("Artifact Store is closed")
782
+
783
+
784
+ class ArtifactBundle:
785
+ """One reserved execution transaction; publish or abort exactly once."""
786
+
787
+ def __init__(
788
+ self,
789
+ store: ArtifactStore,
790
+ identifier: str,
791
+ stage_name: str,
792
+ stage_identity: _Identity,
793
+ ) -> None:
794
+ self._store = store
795
+ self._identifier = identifier
796
+ self._stage_name = stage_name
797
+ self._stage_identity = stage_identity
798
+ self._artifact_ids: set[str] = set()
799
+ self._writers: set[ArtifactWriter] = set()
800
+ self._artifacts: list[_Artifact] = []
801
+ self._parts = 0
802
+ self._charge = 0
803
+ self._settled = False
804
+ self._deletion_progress = _DeletionProgress()
805
+
806
+ @property
807
+ def settled(self) -> bool:
808
+ return self._settled
809
+
810
+ def open_artifact(
811
+ self, name: str, declared_mime_type: str, *, safe_mime_type: str | None = None
812
+ ) -> ArtifactWriter:
813
+ """Open one incremental logical artifact with optional trusted serving MIME."""
814
+ if self._settled:
815
+ raise ArtifactStoreError("bundle is settled")
816
+ if len(self._writers) + len(self._artifacts) >= MAX_STORED_ITEMS:
817
+ raise ArtifactStoreError("execution artifact item limit exceeded")
818
+ validate_logical_name(name)
819
+ validate_mime_type(declared_mime_type)
820
+ if safe_mime_type is not None:
821
+ validate_mime_type(safe_mime_type)
822
+ artifact_id, directory, directory_identity = self._create_artifact_directory()
823
+ self._artifact_ids.add(artifact_id)
824
+ writer = ArtifactWriter(
825
+ self,
826
+ artifact_id,
827
+ directory,
828
+ directory_identity,
829
+ name,
830
+ declared_mime_type,
831
+ safe_mime_type,
832
+ )
833
+ self._writers.add(writer)
834
+ return writer
835
+
836
+ def _create_artifact_directory(self) -> tuple[str, str, _Identity]:
837
+ stage_fd = os.open(
838
+ self._stage_name,
839
+ os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0),
840
+ dir_fd=self._store._root_fd,
841
+ )
842
+ try:
843
+ if _identity(os.fstat(stage_fd)) != self._stage_identity:
844
+ raise ArtifactStoreError("bundle staging identity mismatch")
845
+ for _ in range(_ID_ATTEMPTS):
846
+ identifier = self._store._id_factory()
847
+ if not _valid_id(identifier):
848
+ raise ArtifactStoreError("ID factory returned an invalid UUIDv4")
849
+ name = f"artifact-{identifier}"
850
+ if (
851
+ identifier in self._store._artifacts
852
+ or identifier in self._store._reserved_artifact_ids
853
+ ):
854
+ continue
855
+ try:
856
+ os.mkdir(name, mode=0o700, dir_fd=stage_fd)
857
+ except FileExistsError:
858
+ continue
859
+ os.chmod(name, 0o700, dir_fd=stage_fd, follow_symlinks=False)
860
+ observed = os.stat(name, dir_fd=stage_fd, follow_symlinks=False)
861
+ self._store._reserved_artifact_ids.add(identifier)
862
+ return identifier, name, _identity(observed)
863
+ finally:
864
+ os.close(stage_fd)
865
+ raise ArtifactStoreError("artifact ID collision exhaustion")
866
+
867
+ async def publish(self) -> tuple[ArtifactRecord, ...]:
868
+ """Atomically make every finished item visible."""
869
+ async with self._store._gate:
870
+ if self._settled or self not in self._store._active:
871
+ raise ArtifactStoreError("bundle is settled")
872
+ if self._writers:
873
+ raise ArtifactStoreError("bundle has unfinished artifacts")
874
+ artifacts = {
875
+ artifact.record.uri.rsplit("/", 1)[-1]: artifact for artifact in self._artifacts
876
+ }
877
+ if set(artifacts) != self._artifact_ids or any(
878
+ identifier in self._store._artifacts for identifier in artifacts
879
+ ):
880
+ raise ArtifactStoreError("artifact ID was already published")
881
+ observed_stage = os.stat(
882
+ self._stage_name,
883
+ dir_fd=self._store._root_fd,
884
+ follow_symlinks=False,
885
+ )
886
+ if _identity(observed_stage) != self._stage_identity:
887
+ raise ArtifactStoreError("bundle staging identity mismatch")
888
+ published_name = f"bundle-{self._identifier}"
889
+ try:
890
+ os.stat(published_name, dir_fd=self._store._root_fd, follow_symlinks=False)
891
+ except FileNotFoundError:
892
+ pass
893
+ else:
894
+ raise ArtifactStoreError("published bundle name collision")
895
+ try:
896
+ os.rename(
897
+ self._stage_name,
898
+ published_name,
899
+ src_dir_fd=self._store._root_fd,
900
+ dst_dir_fd=self._store._root_fd,
901
+ )
902
+ except OSError as error:
903
+ self._abort_locked()
904
+ raise ArtifactStoreError("bundle publication failed") from error
905
+ published = os.stat(
906
+ published_name,
907
+ dir_fd=self._store._root_fd,
908
+ follow_symlinks=False,
909
+ )
910
+ if _identity(published) != self._stage_identity:
911
+ raise ArtifactStoreError("published bundle identity mismatch")
912
+ bundle = _Bundle(
913
+ self._identifier,
914
+ published_name,
915
+ artifacts,
916
+ self._stage_identity,
917
+ self._charge,
918
+ )
919
+ self._store._bundles[self._identifier] = bundle
920
+ for artifact_id, artifact in artifacts.items():
921
+ self._store._artifacts[artifact_id] = (bundle, artifact)
922
+ self._store._reserved_artifact_ids.difference_update(self._artifact_ids)
923
+ self._store._retained_charge += self._charge
924
+ self._store._reserved_charge -= self._store._reservation_bytes
925
+ self._store._active.remove(self)
926
+ self._settled = True
927
+ return tuple(artifact.record for artifact in self._artifacts)
928
+
929
+ async def discard(self) -> None:
930
+ """Remove this execution bundle whether staged or already published."""
931
+ async with self._store._gate:
932
+ if not self._settled:
933
+ self._abort_locked()
934
+ return
935
+ deletion = self._store._deleting_bundles.get(self._identifier)
936
+ if deletion is None:
937
+ published = self._store._bundles.get(self._identifier)
938
+ if published is None:
939
+ return
940
+ if published.leases:
941
+ raise ArtifactStoreError("execution bundle is in use")
942
+ deletion = self._store._revoke_bundle(published)
943
+ try:
944
+ self._store._remove_published_bundle(deletion)
945
+ except OSError as error:
946
+ raise ArtifactStoreError("Artifact Store bundle deletion failed") from error
947
+ self._store._complete_deletion(self._identifier, deletion)
948
+
949
+ async def abort(self) -> None:
950
+ async with self._store._gate:
951
+ try:
952
+ self._abort_locked()
953
+ except OSError as error:
954
+ raise ArtifactStoreError("Artifact Store bundle deletion failed") from error
955
+
956
+ def _abort_locked(self) -> None:
957
+ if self._settled:
958
+ return
959
+ progress = self._deletion_progress
960
+ current_stage = progress.tombstones.get("stage", self._stage_name)
961
+ try:
962
+ stage_fd = os.open(
963
+ current_stage,
964
+ os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0),
965
+ dir_fd=self._store._root_fd,
966
+ )
967
+ except OSError as error:
968
+ raise ArtifactStoreError("bundle staging identity mismatch") from error
969
+ try:
970
+ if _identity(os.fstat(stage_fd)) != self._stage_identity:
971
+ raise ArtifactStoreError("bundle staging identity mismatch")
972
+ self._remove_staged_contents(stage_fd)
973
+ self._store._remove_deleting_directory(
974
+ self._store._root_fd,
975
+ self._stage_name,
976
+ self._stage_identity,
977
+ progress,
978
+ "stage",
979
+ )
980
+ finally:
981
+ os.close(stage_fd)
982
+ self._store._active.discard(self)
983
+ self._store._reserved_artifact_ids.difference_update(self._artifact_ids)
984
+ self._store._reserved_charge -= self._store._reservation_bytes
985
+ self._settled = True
986
+
987
+ def _remove_staged_contents(self, stage_fd: int) -> None:
988
+ entries: list[tuple[str, _Identity, dict[str, _Identity]]] = []
989
+ for artifact in self._artifacts:
990
+ files = {part.name: part.identity for part in artifact.parts}
991
+ if artifact.manifest_name is not None:
992
+ assert artifact.manifest_identity is not None
993
+ files[artifact.manifest_name] = artifact.manifest_identity
994
+ entries.append((artifact.directory, artifact.directory_identity, files))
995
+ for writer in self._writers:
996
+ entries.append(
997
+ (
998
+ writer._directory,
999
+ writer._directory_identity,
1000
+ {part.name: part.identity for part in writer._parts},
1001
+ )
1002
+ )
1003
+ progress = self._deletion_progress
1004
+ expected_directories = {
1005
+ progress.tombstones.get(key, directory)
1006
+ for directory, _, _ in entries
1007
+ if (key := f"artifact:{directory}") not in progress.completed
1008
+ }
1009
+ if set(os.listdir(stage_fd)) != expected_directories:
1010
+ raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
1011
+ nofollow = getattr(os, "O_NOFOLLOW", 0)
1012
+ for directory, directory_identity, files in entries:
1013
+ directory_key = f"artifact:{directory}"
1014
+ if directory_key in progress.completed:
1015
+ continue
1016
+ current_directory = progress.tombstones.get(directory_key, directory)
1017
+ artifact_fd = os.open(
1018
+ current_directory,
1019
+ os.O_RDONLY | os.O_DIRECTORY | nofollow,
1020
+ dir_fd=stage_fd,
1021
+ )
1022
+ try:
1023
+ if _identity(os.fstat(artifact_fd)) != directory_identity:
1024
+ raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
1025
+ expected_files = {
1026
+ progress.tombstones.get(key, name)
1027
+ for name in files
1028
+ if (key := f"{directory_key}/file:{name}") not in progress.completed
1029
+ }
1030
+ if set(os.listdir(artifact_fd)) != expected_files:
1031
+ raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
1032
+ for name, file_identity in files.items():
1033
+ self._store._unlink_deleting_file(
1034
+ artifact_fd,
1035
+ name,
1036
+ file_identity,
1037
+ progress,
1038
+ f"{directory_key}/file:{name}",
1039
+ )
1040
+ self._store._remove_deleting_directory(
1041
+ stage_fd,
1042
+ directory,
1043
+ directory_identity,
1044
+ progress,
1045
+ directory_key,
1046
+ )
1047
+ finally:
1048
+ os.close(artifact_fd)
1049
+
1050
+ def _discard(self, writer: ArtifactWriter) -> None:
1051
+ if writer not in self._writers or self._settled:
1052
+ raise ArtifactStoreError("artifact writer is not active")
1053
+ nofollow = getattr(os, "O_NOFOLLOW", 0)
1054
+ progress = self._deletion_progress
1055
+ directory_key = f"artifact:{writer._directory}"
1056
+ try:
1057
+ current_stage = progress.tombstones.get("stage", self._stage_name)
1058
+ stage_fd = os.open(
1059
+ current_stage,
1060
+ os.O_RDONLY | os.O_DIRECTORY | nofollow,
1061
+ dir_fd=self._store._root_fd,
1062
+ )
1063
+ try:
1064
+ if _identity(os.fstat(stage_fd)) != self._stage_identity:
1065
+ raise ArtifactStoreError("bundle staging identity mismatch")
1066
+ current_directory = progress.tombstones.get(directory_key, writer._directory)
1067
+ artifact_fd = os.open(
1068
+ current_directory,
1069
+ os.O_RDONLY | os.O_DIRECTORY | nofollow,
1070
+ dir_fd=stage_fd,
1071
+ )
1072
+ try:
1073
+ if _identity(os.fstat(artifact_fd)) != writer._directory_identity:
1074
+ raise ArtifactStoreError("artifact staging identity mismatch")
1075
+ files = {part.name: part.identity for part in writer._parts}
1076
+ expected_files = {
1077
+ progress.tombstones.get(key, name)
1078
+ for name in files
1079
+ if (key := f"{directory_key}/file:{name}") not in progress.completed
1080
+ }
1081
+ if set(os.listdir(artifact_fd)) != expected_files:
1082
+ raise ArtifactStoreError("Artifact Store cleanup found an unexpected entry")
1083
+ for name, identity in files.items():
1084
+ self._store._unlink_deleting_file(
1085
+ artifact_fd,
1086
+ name,
1087
+ identity,
1088
+ progress,
1089
+ f"{directory_key}/file:{name}",
1090
+ )
1091
+ self._store._remove_deleting_directory(
1092
+ stage_fd,
1093
+ writer._directory,
1094
+ writer._directory_identity,
1095
+ progress,
1096
+ directory_key,
1097
+ )
1098
+ finally:
1099
+ os.close(artifact_fd)
1100
+ finally:
1101
+ os.close(stage_fd)
1102
+ except OSError as error:
1103
+ raise ArtifactStoreError("Artifact Store bundle deletion failed") from error
1104
+ self._writers.remove(writer)
1105
+ self._artifact_ids.remove(writer._artifact_id)
1106
+ self._store._reserved_artifact_ids.remove(writer._artifact_id)
1107
+
1108
+ def _validate_finished(self, writer: ArtifactWriter, part_count: int, charge: int) -> int:
1109
+ if writer not in self._writers or self._settled:
1110
+ raise ArtifactStoreError("artifact writer is not active")
1111
+ new_parts = self._parts + part_count
1112
+ if new_parts > MAX_EXECUTION_PARTS:
1113
+ raise ArtifactStoreError("execution artifact part limit exceeded")
1114
+ if self._charge + charge > self._store._reservation_bytes:
1115
+ raise ArtifactStoreError("execution artifact reservation exceeded")
1116
+ return new_parts
1117
+
1118
+ def _finished(self, writer: ArtifactWriter, artifact: _Artifact, charge: int) -> None:
1119
+ new_parts = self._validate_finished(writer, len(artifact.parts), charge)
1120
+ self._writers.remove(writer)
1121
+ self._artifacts.append(artifact)
1122
+ self._parts = new_parts
1123
+ self._charge += charge
1124
+
1125
+
1126
+ class ArtifactWriter:
1127
+ """Incrementally hash and split one logical artifact into immutable parts."""
1128
+
1129
+ def __init__(
1130
+ self,
1131
+ bundle: ArtifactBundle,
1132
+ artifact_id: str,
1133
+ directory: str,
1134
+ directory_identity: _Identity,
1135
+ name: str,
1136
+ declared_mime_type: str,
1137
+ safe_mime_type: str | None,
1138
+ ) -> None:
1139
+ self._bundle = bundle
1140
+ self._artifact_id = artifact_id
1141
+ self._directory = directory
1142
+ self._directory_identity = directory_identity
1143
+ self._name = name
1144
+ self._declared_mime_type = declared_mime_type
1145
+ self._safe_mime_type = safe_mime_type
1146
+ self._hash = hashlib.sha256()
1147
+ self._size = 0
1148
+ self._parts: list[_Part] = []
1149
+ self._buffer: bytearray | None = None
1150
+ self._buffer_size = 0
1151
+ self._finished = False
1152
+
1153
+ def write(self, data: bytes | bytearray | memoryview) -> None:
1154
+ if self._finished:
1155
+ raise ArtifactStoreError("artifact writer is finished")
1156
+ view = memoryview(data)
1157
+ if self._size + len(view) > MAX_ARTIFACT_BYTES:
1158
+ raise ArtifactStoreError("artifact byte limit exceeded")
1159
+ self._hash.update(view)
1160
+ self._size += len(view)
1161
+ while view:
1162
+ if self._buffer is None:
1163
+ self._buffer = bytearray(ARTIFACT_RESOURCE_PART_BYTES)
1164
+ available = ARTIFACT_RESOURCE_PART_BYTES - self._buffer_size
1165
+ take = min(len(view), available)
1166
+ self._buffer[self._buffer_size : self._buffer_size + take] = view[:take]
1167
+ self._buffer_size += take
1168
+ view = view[take:]
1169
+ if self._buffer_size == ARTIFACT_RESOURCE_PART_BYTES:
1170
+ self._write_part()
1171
+
1172
+ def discard(self) -> None:
1173
+ """Remove one unfinished logical artifact from its active bundle."""
1174
+ if self._finished:
1175
+ raise ArtifactStoreError("artifact writer is finished")
1176
+ try:
1177
+ self._bundle._discard(self)
1178
+ except OSError as error:
1179
+ raise ArtifactStoreError("artifact discard failed") from error
1180
+ self._buffer = None
1181
+ self._buffer_size = 0
1182
+ self._finished = True
1183
+
1184
+ def _set_safe_mime_type(self, mime_type: str) -> None:
1185
+ """Promote serving MIME after Host content validation."""
1186
+ if self._finished:
1187
+ raise ArtifactStoreError("artifact writer is finished")
1188
+ validate_mime_type(mime_type)
1189
+ self._safe_mime_type = mime_type
1190
+
1191
+ def finish(self) -> ArtifactRecord:
1192
+ if self._finished:
1193
+ raise ArtifactStoreError("artifact writer is finished")
1194
+ if self._buffer_size or not self._parts:
1195
+ self._write_part()
1196
+ if len(self._parts) > MAX_ARTIFACT_PARTS:
1197
+ raise ArtifactStoreError("artifact part limit exceeded")
1198
+ charge = self._size + _ITEM_CHARGE + len(self._parts) * _PART_CHARGE
1199
+ self._bundle._validate_finished(self, len(self._parts), charge)
1200
+ digest = self._hash.hexdigest()
1201
+ uri = f"{_URI_PREFIX}{self._bundle._store.server_id}/{self._artifact_id}"
1202
+ representation: Literal["raw", "manifest"] = (
1203
+ "raw" if self._size <= ARTIFACT_RESOURCE_PART_BYTES else "manifest"
1204
+ )
1205
+ mime_type = (
1206
+ self._safe_mime_type or _safe_mime(self._declared_mime_type)
1207
+ if representation == "raw"
1208
+ else _MANIFEST_MIME
1209
+ )
1210
+ manifest_name: str | None = None
1211
+ if representation == "manifest":
1212
+ manifest = {
1213
+ "version": 1,
1214
+ "name": self._name,
1215
+ "declaredMimeType": self._declared_mime_type,
1216
+ "size": self._size,
1217
+ "sha256": digest,
1218
+ "parts": [
1219
+ {
1220
+ "uri": f"{uri}/part/{index}",
1221
+ "offset": part.offset,
1222
+ "size": part.size,
1223
+ "sha256": part.sha256,
1224
+ }
1225
+ for index, part in enumerate(self._parts)
1226
+ ],
1227
+ }
1228
+ encoded = json.dumps(
1229
+ manifest,
1230
+ ensure_ascii=False,
1231
+ allow_nan=False,
1232
+ separators=(",", ":"),
1233
+ ).encode("utf-8")
1234
+ if len(encoded) >= _MAX_MANIFEST_BYTES:
1235
+ raise ArtifactStoreError("artifact manifest exceeds its bound")
1236
+ manifest_name = "manifest.json"
1237
+ manifest_size = len(encoded)
1238
+ manifest_sha256 = hashlib.sha256(encoded).hexdigest()
1239
+ manifest_identity = self._write_atomic_file(manifest_name, encoded)
1240
+ else:
1241
+ manifest_size = None
1242
+ manifest_sha256 = None
1243
+ manifest_identity = None
1244
+ record = ArtifactRecord(
1245
+ uri=uri,
1246
+ name=self._name,
1247
+ declared_mime_type=self._declared_mime_type,
1248
+ mime_type=mime_type,
1249
+ representation=representation,
1250
+ size=self._size,
1251
+ sha256=digest,
1252
+ )
1253
+ artifact = _Artifact(
1254
+ record,
1255
+ self._directory,
1256
+ self._directory_identity,
1257
+ tuple(self._parts),
1258
+ manifest_name,
1259
+ manifest_size,
1260
+ manifest_sha256,
1261
+ manifest_identity,
1262
+ )
1263
+ self._bundle._finished(self, artifact, charge)
1264
+ self._finished = True
1265
+ return record
1266
+
1267
+ def _write_part(self) -> None:
1268
+ if len(self._parts) >= MAX_ARTIFACT_PARTS:
1269
+ raise ArtifactStoreError("artifact part limit exceeded")
1270
+ if self._buffer is None:
1271
+ self._buffer = bytearray(ARTIFACT_RESOURCE_PART_BYTES)
1272
+ data = memoryview(self._buffer)[: self._buffer_size]
1273
+ self._buffer_size = 0
1274
+ index = len(self._parts)
1275
+ name = f"part-{index}"
1276
+ identity = self._write_atomic_file(name, data)
1277
+ self._parts.append(
1278
+ _Part(
1279
+ name=name,
1280
+ offset=index * ARTIFACT_RESOURCE_PART_BYTES,
1281
+ size=len(data),
1282
+ sha256=hashlib.sha256(data).hexdigest(),
1283
+ identity=identity,
1284
+ )
1285
+ )
1286
+
1287
+ def _write_atomic_file(
1288
+ self, final_name: str, data: bytes | bytearray | memoryview
1289
+ ) -> _Identity:
1290
+ nofollow = getattr(os, "O_NOFOLLOW", 0)
1291
+ stage_fd = os.open(
1292
+ self._bundle._stage_name,
1293
+ os.O_RDONLY | os.O_DIRECTORY | nofollow,
1294
+ dir_fd=self._bundle._store._root_fd,
1295
+ )
1296
+ try:
1297
+ if _identity(os.fstat(stage_fd)) != self._bundle._stage_identity:
1298
+ raise ArtifactStoreError("bundle staging identity mismatch")
1299
+ artifact_fd = os.open(
1300
+ self._directory,
1301
+ os.O_RDONLY | os.O_DIRECTORY | nofollow,
1302
+ dir_fd=stage_fd,
1303
+ )
1304
+ try:
1305
+ if _identity(os.fstat(artifact_fd)) != self._directory_identity:
1306
+ raise ArtifactStoreError("artifact staging identity mismatch")
1307
+ temporary = f"tmp-{uuid.uuid4().hex}"
1308
+ descriptor = os.open(
1309
+ temporary,
1310
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL | nofollow,
1311
+ 0o600,
1312
+ dir_fd=artifact_fd,
1313
+ )
1314
+ try:
1315
+ view = memoryview(data)
1316
+ while view:
1317
+ written = os.write(descriptor, view)
1318
+ if written <= 0:
1319
+ raise ArtifactStoreError("artifact write made no progress")
1320
+ view = view[written:]
1321
+ os.fsync(descriptor)
1322
+ observed = os.fstat(descriptor)
1323
+ if observed.st_size != len(data):
1324
+ raise ArtifactStoreError("artifact write size mismatch")
1325
+ os.fchmod(descriptor, 0o400)
1326
+ file_identity = _identity(observed)
1327
+ except BaseException:
1328
+ try:
1329
+ os.unlink(temporary, dir_fd=artifact_fd)
1330
+ except OSError:
1331
+ pass
1332
+ raise
1333
+ finally:
1334
+ os.close(descriptor)
1335
+ try:
1336
+ try:
1337
+ os.stat(final_name, dir_fd=artifact_fd, follow_symlinks=False)
1338
+ except FileNotFoundError:
1339
+ pass
1340
+ else:
1341
+ raise ArtifactStoreError("artifact file already exists")
1342
+ os.rename(
1343
+ temporary,
1344
+ final_name,
1345
+ src_dir_fd=artifact_fd,
1346
+ dst_dir_fd=artifact_fd,
1347
+ )
1348
+ published = os.stat(final_name, dir_fd=artifact_fd, follow_symlinks=False)
1349
+ if _identity(published) != file_identity:
1350
+ raise ArtifactStoreError("artifact file identity mismatch")
1351
+ return file_identity
1352
+ except BaseException:
1353
+ try:
1354
+ os.unlink(temporary, dir_fd=artifact_fd)
1355
+ except OSError:
1356
+ pass
1357
+ raise
1358
+ finally:
1359
+ os.close(artifact_fd)
1360
+ except OSError as error:
1361
+ raise ArtifactStoreError("artifact write failed") from error
1362
+ finally:
1363
+ os.close(stage_fd)