kd-engine 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. kd_engine-1.0.0.dist-info/METADATA +19 -0
  2. kd_engine-1.0.0.dist-info/RECORD +118 -0
  3. kd_engine-1.0.0.dist-info/WHEEL +5 -0
  4. kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
  5. kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
  6. kd_engine-1.0.0.dist-info/top_level.txt +1 -0
  7. ytk/__init__.py +2 -0
  8. ytk/__main__.py +21 -0
  9. ytk/batch/__init__.py +39 -0
  10. ytk/batch/runner.py +1149 -0
  11. ytk/cache/__init__.py +34 -0
  12. ytk/cache/v2.py +562 -0
  13. ytk/cleaning/density.py +40 -0
  14. ytk/cleaning/fillers.py +18 -0
  15. ytk/cleaning/sentence_boundary.py +30 -0
  16. ytk/cleaning/service.py +62 -0
  17. ytk/cli.py +1281 -0
  18. ytk/commands/__init__.py +3 -0
  19. ytk/commands/merge.py +600 -0
  20. ytk/config.py +757 -0
  21. ytk/dedup/service.py +166 -0
  22. ytk/doctor.py +891 -0
  23. ytk/dotenv.py +334 -0
  24. ytk/errors.py +38 -0
  25. ytk/extraction/service.py +451 -0
  26. ytk/extraction/strategies/base.py +16 -0
  27. ytk/extraction/strategies/code_block.py +285 -0
  28. ytk/extraction/strategies/heuristic.py +791 -0
  29. ytk/extraction/strategies/local_model.py +56 -0
  30. ytk/extraction/text_context.py +177 -0
  31. ytk/extraction/visual_context.py +159 -0
  32. ytk/ingestion/adapters/__init__.py +1 -0
  33. ytk/ingestion/adapters/_common.py +865 -0
  34. ytk/ingestion/adapters/base.py +160 -0
  35. ytk/ingestion/adapters/bilibili.py +271 -0
  36. ytk/ingestion/adapters/douyin.py +445 -0
  37. ytk/ingestion/adapters/tiktok.py +102 -0
  38. ytk/ingestion/adapters/xiaohongshu.py +758 -0
  39. ytk/ingestion/adapters/youtube.py +74 -0
  40. ytk/ingestion/browser_acquired.py +220 -0
  41. ytk/ingestion/http_client.py +43 -0
  42. ytk/ingestion/local_import.py +202 -0
  43. ytk/ingestion/media.py +468 -0
  44. ytk/ingestion/providers/asr.py +902 -0
  45. ytk/ingestion/providers/base.py +13 -0
  46. ytk/ingestion/providers/dtk_client.py +816 -0
  47. ytk/ingestion/providers/minimax.py +350 -0
  48. ytk/ingestion/providers/transcript_api.py +336 -0
  49. ytk/ingestion/providers/xhs_client.py +646 -0
  50. ytk/ingestion/providers/ytdlp.py +400 -0
  51. ytk/ingestion/registry.py +189 -0
  52. ytk/ingestion/resolver.py +417 -0
  53. ytk/ingestion/search.py +56 -0
  54. ytk/ingestion/service.py +167 -0
  55. ytk/ingestion/subtitles.py +602 -0
  56. ytk/ingestion/transcript_cache.py +212 -0
  57. ytk/ingestion/url_parser.py +53 -0
  58. ytk/ingestion/video_frames.py +292 -0
  59. ytk/ingestion/ytdlp_acquire.py +754 -0
  60. ytk/logging.py +383 -0
  61. ytk/models/common.py +20 -0
  62. ytk/models/content.py +451 -0
  63. ytk/models/knowledge.py +43 -0
  64. ytk/models/migrations.py +262 -0
  65. ytk/models/skill.py +13 -0
  66. ytk/models/source.py +265 -0
  67. ytk/models/topic.py +18 -0
  68. ytk/models/transcript.py +30 -0
  69. ytk/ocr/__init__.py +65 -0
  70. ytk/ocr/engine.py +245 -0
  71. ytk/ocr/layout.py +347 -0
  72. ytk/ocr/pipeline.py +440 -0
  73. ytk/ocr/rapidocr_engine.py +144 -0
  74. ytk/pipeline/acquisition_checkpoint.py +240 -0
  75. ytk/pipeline/artifacts.py +18 -0
  76. ytk/pipeline/orchestrator.py +598 -0
  77. ytk/pipeline/processing_identity.py +18 -0
  78. ytk/pipeline/retry.py +728 -0
  79. ytk/pipeline/runners.py +578 -0
  80. ytk/pipeline/state.py +1107 -0
  81. ytk/pipeline/v2.py +1863 -0
  82. ytk/py.typed +1 -0
  83. ytk/quality/__init__.py +9 -0
  84. ytk/quality/completeness.py +260 -0
  85. ytk/renderers/base.py +9 -0
  86. ytk/renderers/citation_alignment.py +617 -0
  87. ytk/renderers/evidence_formatter.py +362 -0
  88. ytk/renderers/json_renderer.py +13 -0
  89. ytk/renderers/jsonl_renderer.py +18 -0
  90. ytk/renderers/knowledge_doc_renderer.py +1984 -0
  91. ytk/renderers/markdown_renderer.py +540 -0
  92. ytk/renderers/obsidian_renderer.py +138 -0
  93. ytk/renderers/skill_package.py +21 -0
  94. ytk/renderers/source_links.py +309 -0
  95. ytk/renderers/summary_renderer.py +163 -0
  96. ytk/renderers/v2.py +171 -0
  97. ytk/renderers/v2_jsonl.py +165 -0
  98. ytk/renderers/v2_markdown.py +477 -0
  99. ytk/renderers/v2_obsidian.py +113 -0
  100. ytk/renderers/v2_summary.py +220 -0
  101. ytk/segmentation/change_points.py +50 -0
  102. ytk/segmentation/embeddings.py +45 -0
  103. ytk/segmentation/service.py +318 -0
  104. ytk/skill_builder/__init__.py +1 -0
  105. ytk/skill_builder/builder.py +419 -0
  106. ytk/skill_builder/templates/SKILL.md.j2 +85 -0
  107. ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
  108. ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
  109. ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
  110. ytk/skill_builder/templates/references/commands.md.j2 +34 -0
  111. ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
  112. ytk/skill_builder/templates/references/configs.md.j2 +31 -0
  113. ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
  114. ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
  115. ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
  116. ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
  117. ytk/skill_builder/templates.py +29 -0
  118. ytk/text_normalization.py +44 -0
ytk/pipeline/state.py ADDED
@@ -0,0 +1,1107 @@
1
+ """T12.2 — v2 artifact manifest, atomic writes and restart decisions.
2
+
3
+ Frozen by ``docs/implementation/five-platform/contracts-v2.md`` §1.4:
4
+
5
+ * :class:`ManifestV2` records source identity, the acquisition revision, the
6
+ request fingerprint, per-stage status, per-format outputs, missing items,
7
+ remote tasks and the refresh epoch. Extra keys are accepted so the v2
8
+ pipeline can keep its ``outcome``/``reasons``/``refresh`` diagnostics in the
9
+ same file (the v2 pipeline treats ``outcome`` as authoritative and the stage
10
+ map as diagnostic).
11
+ * :class:`ArtifactStore` owns every write into one task directory: locking
12
+ (``filelock`` when installed, stdlib ``fcntl.flock`` fallback, per-path
13
+ reentrant), temp-file + validate + ``os.replace`` atomic writes, and the
14
+ manifest **last**.
15
+ * A corrupt/missing output or manifest can never be used to resume: digest and
16
+ validator-version mismatches fail closed. A task directory owned by another
17
+ source is refused (different works must not share ``out``).
18
+ * ``unknown`` only ever maps to an overall ``blocked`` outcome with a reason
19
+ code; an unresolved remote submission (DISPATCHING / unknown task id) can
20
+ never be bypassed by changing fingerprints or refreshing.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import hashlib
26
+ import json
27
+ import logging
28
+ import os
29
+ import tempfile
30
+ import threading
31
+ import time
32
+ from collections.abc import Iterator, Mapping, Sequence
33
+ from contextlib import contextmanager
34
+ from datetime import UTC, datetime
35
+ from pathlib import Path
36
+ from typing import Any, Literal
37
+
38
+ from pydantic import BaseModel, ConfigDict, Field, ValidationError
39
+
40
+ LOGGER = logging.getLogger(__name__)
41
+
42
+ MANIFEST_FILENAME = "manifest.json"
43
+ MANIFEST_SCHEMA_VERSION = 2
44
+
45
+ StageName = Literal[
46
+ "resolve",
47
+ "acquire",
48
+ "subtitle",
49
+ "media",
50
+ "asr",
51
+ "ocr",
52
+ "segment",
53
+ "extract",
54
+ "render",
55
+ ]
56
+ StageStatus = Literal["pending", "running", "complete", "partial", "blocked", "failed", "unknown"]
57
+ OutputStatus = Literal[
58
+ "pending", "complete", "partial", "failed", "blocked", "not_applicable", "skipped"
59
+ ]
60
+
61
+ #: Overall outcomes (plan §4.5 / contract §1.4).
62
+ OVERALL_COMPLETE = "COMPLETE"
63
+ OVERALL_PARTIAL = "PARTIAL"
64
+ OVERALL_BLOCKED = "BLOCKED"
65
+ OVERALL_FAILED = "FAILED"
66
+ OVERALL_NO_KNOWLEDGE = "NO_KNOWLEDGE"
67
+
68
+ #: Reason codes used by the resume/refresh machinery and the CLI exit mapping.
69
+ REASON_REMOTE_SUBMISSION_UNKNOWN = "REMOTE_SUBMISSION_UNKNOWN"
70
+ REASON_DISPATCHING_UNKNOWN = "DISPATCHING_UNKNOWN"
71
+ REASON_UNKNOWN_STAGE = "UNKNOWN_STAGE"
72
+ REASON_OUTPUTS_NOT_APPLICABLE = "OUTPUTS_NOT_APPLICABLE"
73
+ REASON_NO_KNOWLEDGE = "NO_KNOWLEDGE"
74
+ REASON_REFRESH = "REFRESH"
75
+ REASON_ARTIFACT_CORRUPT = "ARTIFACT_CORRUPT"
76
+ REASON_ARTIFACT_MISSING = "ARTIFACT_MISSING"
77
+ REASON_VALIDATOR_MISMATCH = "VALIDATOR_MISMATCH"
78
+
79
+ #: Output validator version written by the v2 pipeline. It doubles as the
80
+ #: judgment-policy version when an entry records ``validator_version`` but no
81
+ #: explicit ``checker_version`` (R-05/R-11).
82
+ OUTPUT_VALIDATOR_VERSION = "v2-1"
83
+
84
+ #: Remote submission states that are *not resolved* yet. A task in one of
85
+ #: these states must never be re-submitted or hidden behind a new fingerprint.
86
+ UNRESOLVED_REMOTE_STATUSES: frozenset[str] = frozenset(
87
+ {"dispatching", "unknown", "pending", "running", "in_progress", "queued", "submitted"}
88
+ )
89
+ #: Remote states that *are* resolved (success or a terminal failure).
90
+ RESOLVED_REMOTE_STATUSES: frozenset[str] = frozenset(
91
+ {"complete", "completed", "done", "succeeded", "failed", "cancelled", "canceled",
92
+ "expired", "resolved", "not_required"}
93
+ )
94
+
95
+ #: Stages that depend on acquisition/remote work (privileged for refresh).
96
+ _ACQUISITION_STAGES: frozenset[str] = frozenset(
97
+ {"resolve", "acquire", "subtitle", "media", "asr", "ocr"}
98
+ )
99
+
100
+ _LOCK_SUFFIX = ".lock"
101
+
102
+
103
+ # --------------------------------------------------------------------------- #
104
+ # Locking: filelock → fcntl fallback → process-local
105
+ # --------------------------------------------------------------------------- #
106
+ try: # optional dependency (``lock`` extra); never required by a base install
107
+ import filelock as _filelock
108
+
109
+ _FILELOCK_AVAILABLE = True
110
+ except ImportError: # pragma: no cover - exercised by the fallback test
111
+ _filelock = None # type: ignore[assignment]
112
+ _FILELOCK_AVAILABLE = False
113
+
114
+ try: # POSIX fallback
115
+ import fcntl as _fcntl
116
+
117
+ _FCNTL_AVAILABLE = True
118
+ except ImportError: # pragma: no cover - Windows without pywin32
119
+ _fcntl = None # type: ignore[assignment]
120
+ _FCNTL_AVAILABLE = False
121
+
122
+
123
+ class ArtifactStoreError(RuntimeError):
124
+ """Base error for artifact-store failures (never silent data corruption)."""
125
+
126
+
127
+ class ArtifactLockTimeoutError(ArtifactStoreError):
128
+ """Another process/thread holds the task directory lock for too long."""
129
+
130
+
131
+ class ArtifactOwnershipError(ArtifactStoreError):
132
+ """A different work already owns this output directory — refusing to mix."""
133
+
134
+
135
+ class UnresolvedRemoteSubmissionError(ArtifactStoreError):
136
+ """A remote job may still be running; never bypass it by changing state."""
137
+
138
+
139
+ class _ReentrantPathLock:
140
+ """Process-wide reentrant lock for one lock path, with an OS-level guard.
141
+
142
+ Multiple :class:`ArtifactStore` instances (and multiple threads) for the
143
+ same resolved path share one object, so nested use inside one process is
144
+ reentrant instead of self-deadlocking, while a second *process* still waits
145
+ on the OS lock.
146
+ """
147
+
148
+ _registry: dict[str, _ReentrantPathLock] = {}
149
+ _registry_guard = threading.Lock()
150
+
151
+ def __init__(self, path: Path) -> None:
152
+ self.path = path
153
+ self._mutex = threading.RLock()
154
+ self._depth = 0
155
+ self._file_handle: Any = None
156
+ self._filelock_handle: Any = None
157
+ self._backend: str = "none"
158
+
159
+ @classmethod
160
+ def for_path(cls, path: Path) -> _ReentrantPathLock:
161
+ key = str(path.expanduser().resolve())
162
+ with cls._registry_guard:
163
+ existing = cls._registry.get(key)
164
+ if existing is not None:
165
+ return existing
166
+ expanded = path.expanduser()
167
+ created = cls(expanded)
168
+ cls._registry[str(expanded.resolve())] = created
169
+ return created
170
+
171
+ # ------------------------------------------------------------ OS backend
172
+ def _acquire_os_lock(self, timeout: float | None) -> None:
173
+ self.path.parent.mkdir(parents=True, exist_ok=True)
174
+ deadline = None if timeout is None else time.monotonic() + max(0.0, timeout)
175
+ wait = 0.02
176
+ while True:
177
+ if _FILELOCK_AVAILABLE:
178
+ try:
179
+ handle = _filelock.FileLock(str(self.path), timeout=0)
180
+ except TypeError: # very old filelock
181
+ handle = _filelock.FileLock(str(self.path))
182
+ try:
183
+ handle.acquire(timeout=0)
184
+ except _filelock.Timeout:
185
+ if deadline is not None and time.monotonic() >= deadline:
186
+ raise ArtifactLockTimeoutError(
187
+ f"timed out waiting for artifact lock {self.path} "
188
+ f"({timeout}s); another YTK task is writing this directory"
189
+ ) from None
190
+ time.sleep(wait)
191
+ continue
192
+ self._filelock_handle = handle
193
+ self._backend = "filelock"
194
+ return
195
+ if _FCNTL_AVAILABLE:
196
+ file_handle = open(self.path, "a+", encoding="utf-8") # noqa: SIM115 - held
197
+ try:
198
+ _fcntl.flock(file_handle.fileno(), _fcntl.LOCK_EX | _fcntl.LOCK_NB)
199
+ except OSError:
200
+ file_handle.close()
201
+ if deadline is not None and time.monotonic() >= deadline:
202
+ raise ArtifactLockTimeoutError(
203
+ f"timed out waiting for artifact lock {self.path} "
204
+ f"({timeout}s); another YTK task is writing this directory"
205
+ ) from None
206
+ time.sleep(wait)
207
+ continue
208
+ self._file_handle = file_handle
209
+ self._backend = "fcntl"
210
+ return
211
+ # Last resort (no filelock, no fcntl): process-local only.
212
+ self._backend = "process"
213
+ LOGGER.warning(
214
+ "no filelock/fcntl available: artifact lock %s is process-local only",
215
+ self.path,
216
+ )
217
+ return
218
+
219
+ def _release_os_lock(self) -> None:
220
+ if self._filelock_handle is not None:
221
+ try:
222
+ self._filelock_handle.release()
223
+ except Exception: # noqa: BLE001 - release must never raise
224
+ LOGGER.warning("failed to release filelock %s", self.path)
225
+ self._filelock_handle = None
226
+ if self._file_handle is not None:
227
+ try:
228
+ _fcntl.flock(self._file_handle.fileno(), _fcntl.LOCK_UN)
229
+ except OSError:
230
+ pass
231
+ try:
232
+ self._file_handle.close()
233
+ except OSError:
234
+ pass
235
+ self._file_handle = None
236
+ self._backend = "none"
237
+
238
+ # ------------------------------------------------------------ public API
239
+ @contextmanager
240
+ def acquire(self, timeout: float | None) -> Iterator[None]:
241
+ acquired = False
242
+ if timeout is None or timeout < 0:
243
+ self._mutex.acquire()
244
+ acquired = True
245
+ else:
246
+ acquired = self._mutex.acquire(timeout=max(0.0, float(timeout)))
247
+ if not acquired:
248
+ raise ArtifactLockTimeoutError(
249
+ f"timed out waiting for artifact lock {self.path} ({timeout}s)"
250
+ )
251
+ try:
252
+ if self._depth == 0:
253
+ self._acquire_os_lock(timeout)
254
+ self._depth += 1
255
+ try:
256
+ yield
257
+ finally:
258
+ self._depth -= 1
259
+ if self._depth == 0:
260
+ self._release_os_lock()
261
+ finally:
262
+ self._mutex.release()
263
+
264
+
265
+ # --------------------------------------------------------------------------- #
266
+ # Small helpers
267
+ # --------------------------------------------------------------------------- #
268
+ def utc_now() -> str:
269
+ """ISO-8601 UTC timestamp with second precision (manifest ``updated_at``)."""
270
+ return datetime.now(UTC).isoformat(timespec="seconds")
271
+
272
+
273
+ def sha256_bytes(data: bytes) -> str:
274
+ return hashlib.sha256(data).hexdigest()
275
+
276
+
277
+ def sha256_file(path: Path) -> str:
278
+ digest = hashlib.sha256()
279
+ with Path(path).open("rb") as handle:
280
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
281
+ digest.update(chunk)
282
+ return digest.hexdigest()
283
+
284
+
285
+ def directory_file_count(path: Path) -> int:
286
+ """Deterministic file count for a directory artifact (skill packages)."""
287
+ try:
288
+ return sum(1 for item in Path(path).rglob("*") if item.is_file())
289
+ except OSError:
290
+ return -1
291
+
292
+
293
+ def directory_digest(path: Path) -> str | None:
294
+ """Hash package-relative filenames and bytes; never follow symlinks."""
295
+ root = Path(path)
296
+ if root.is_symlink() or not root.is_dir():
297
+ return None
298
+ digest = hashlib.sha256()
299
+ files = 0
300
+ try:
301
+ for item in sorted(root.rglob("*")):
302
+ if item.is_symlink():
303
+ return None
304
+ if not item.is_file():
305
+ continue
306
+ digest.update(item.relative_to(root).as_posix().encode("utf-8"))
307
+ digest.update(b"\0")
308
+ digest.update(bytes.fromhex(sha256_file(item)))
309
+ files += 1
310
+ except (OSError, ValueError):
311
+ return None
312
+ return digest.hexdigest() if files else None
313
+
314
+
315
+ def _entry_extra(entry: OutputEntry) -> Mapping[str, Any]:
316
+ return entry.model_extra or {}
317
+
318
+
319
+ def _entry_is_directory_artifact(entry: OutputEntry, target: Path | None) -> bool:
320
+ kind = _entry_extra(entry).get("kind")
321
+ if kind == "directory":
322
+ return True
323
+ return target is not None and target.is_dir()
324
+
325
+
326
+ def atomic_write_bytes(path: Path, data: bytes) -> None:
327
+ """Write *data* to *path* atomically (same-directory temp + ``os.replace``)."""
328
+ path = Path(path)
329
+ path.parent.mkdir(parents=True, exist_ok=True)
330
+ fd, tmp_name = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".tmp")
331
+ try:
332
+ with os.fdopen(fd, "wb") as handle:
333
+ handle.write(data)
334
+ handle.flush()
335
+ try:
336
+ os.fsync(handle.fileno())
337
+ except OSError: # e.g. some network/CI filesystems
338
+ pass
339
+ os.replace(tmp_name, path)
340
+ except BaseException:
341
+ try:
342
+ os.unlink(tmp_name)
343
+ except OSError:
344
+ pass
345
+ raise
346
+
347
+
348
+ def _jsonable(payload: object) -> object:
349
+ if isinstance(payload, BaseModel):
350
+ return payload.model_dump(mode="json")
351
+ return payload
352
+
353
+
354
+ def _dump_json(payload: object) -> bytes:
355
+ data = json.dumps(_jsonable(payload), indent=2, ensure_ascii=False, sort_keys=True)
356
+ return (data + "\n").encode("utf-8")
357
+
358
+
359
+ # --------------------------------------------------------------------------- #
360
+ # Models
361
+ # --------------------------------------------------------------------------- #
362
+ class OutputEntry(BaseModel):
363
+ """Per-format output record (contract §1.4 / plan §4.5)."""
364
+
365
+ model_config = ConfigDict(extra="allow")
366
+
367
+ status: OutputStatus = "pending"
368
+ path: str | None = None
369
+ sha256: str | None = None
370
+ validator_version: str | None = None
371
+ # ``not_applicable`` records its justification so resume can decide whether
372
+ # the judgment is still valid (reason + evidence fingerprint + checker ver.).
373
+ reason: str | None = None
374
+ reason_code: str | None = None
375
+ evidence_fingerprint: str | None = None
376
+ checker_version: str | None = None
377
+ updated_at: str | None = None
378
+
379
+
380
+ class ManifestV2(BaseModel):
381
+ """The v2 task manifest; written last, never trusts an incomplete run."""
382
+
383
+ model_config = ConfigDict(extra="allow") # pipeline diagnostics ride along
384
+
385
+ schema_version: Literal[2] = 2
386
+ source_ids: list[str] = Field(default_factory=list)
387
+ source_revision: str | None = None
388
+ request_fingerprint: str = ""
389
+ stages: dict[str, StageStatus] = Field(default_factory=dict)
390
+ outputs: dict[str, OutputEntry] = Field(default_factory=dict)
391
+ missing: list[str] = Field(default_factory=list)
392
+ remote_tasks: dict[str, dict[str, Any]] = Field(default_factory=dict)
393
+ updated_at: str = ""
394
+ refresh_epoch: int = 0
395
+
396
+
397
+ # --------------------------------------------------------------------------- #
398
+ # Pure manifest helpers
399
+ # --------------------------------------------------------------------------- #
400
+ def new_manifest(
401
+ *,
402
+ source_ids: Sequence[str],
403
+ request_fingerprint: str,
404
+ source_revision: str | None = None,
405
+ refresh_epoch: int = 0,
406
+ ) -> ManifestV2:
407
+ return ManifestV2(
408
+ source_ids=list(source_ids),
409
+ source_revision=source_revision,
410
+ request_fingerprint=request_fingerprint,
411
+ updated_at=utc_now(),
412
+ refresh_epoch=refresh_epoch,
413
+ )
414
+
415
+
416
+ def mark_stage(manifest: ManifestV2, stage: str, status: StageStatus) -> ManifestV2:
417
+ updated = manifest.model_copy(deep=True)
418
+ updated.stages[stage] = status
419
+ return updated
420
+
421
+
422
+ def record_output(
423
+ manifest: ManifestV2,
424
+ fmt: str,
425
+ *,
426
+ status: OutputStatus,
427
+ path: str | None = None,
428
+ sha256: str | None = None,
429
+ validator_version: str | None = None,
430
+ reason: str | None = None,
431
+ reason_code: str | None = None,
432
+ evidence_fingerprint: str | None = None,
433
+ checker_version: str | None = None,
434
+ ) -> ManifestV2:
435
+ updated = manifest.model_copy(deep=True)
436
+ updated.outputs[fmt] = OutputEntry(
437
+ status=status,
438
+ path=path,
439
+ sha256=sha256,
440
+ validator_version=validator_version,
441
+ reason=reason,
442
+ reason_code=reason_code,
443
+ evidence_fingerprint=evidence_fingerprint,
444
+ checker_version=checker_version,
445
+ updated_at=utc_now(),
446
+ )
447
+ return updated
448
+
449
+
450
+ def record_not_applicable(
451
+ manifest: ManifestV2,
452
+ fmt: str,
453
+ *,
454
+ reason_code: str,
455
+ evidence_fingerprint: str,
456
+ checker_version: str,
457
+ reason: str | None = None,
458
+ ) -> ManifestV2:
459
+ """Record a format-level NOT_APPLICABLE with its justification."""
460
+ return record_output(
461
+ manifest,
462
+ fmt,
463
+ status="not_applicable",
464
+ reason=reason,
465
+ reason_code=reason_code,
466
+ evidence_fingerprint=evidence_fingerprint,
467
+ checker_version=checker_version,
468
+ )
469
+
470
+
471
+ def unresolved_remote_tasks(
472
+ manifest: ManifestV2, provider: str | None = None
473
+ ) -> dict[str, dict[str, Any]]:
474
+ """Remote tasks that may still be running (DISPATCHING / unknown id)."""
475
+ unresolved: dict[str, dict[str, Any]] = {}
476
+ for name, task in manifest.remote_tasks.items():
477
+ if provider is not None and name != provider:
478
+ continue
479
+ if not isinstance(task, Mapping):
480
+ unresolved[name] = {"status": "unknown"}
481
+ continue
482
+ status = str(task.get("status") or "unknown").strip().lower()
483
+ task_id = task.get("task_id")
484
+ if status in UNRESOLVED_REMOTE_STATUSES or (task_id in (None, "") and status not in RESOLVED_REMOTE_STATUSES):
485
+ unresolved[name] = dict(task)
486
+ return unresolved
487
+
488
+
489
+ def _upper_reason(entry: OutputEntry) -> str:
490
+ return str(entry.reason_code or entry.reason or "").strip().upper()
491
+
492
+
493
+ def output_entry_claims_no_knowledge(entry: OutputEntry) -> bool:
494
+ """True when a ``not_applicable`` entry records a NO_KNOWLEDGE verdict.
495
+
496
+ The v2 pipeline records ``reason_code="NO_KNOWLEDGE"`` for the content-level
497
+ no-knowledge verdict (R-11); older manifests may carry the legacy
498
+ ``reason="NO_KNOWLEDGE: …"`` text instead.
499
+ """
500
+ return _upper_reason(entry).startswith(REASON_NO_KNOWLEDGE)
501
+
502
+
503
+ def manifest_recorded_outcome(manifest: ManifestV2) -> str:
504
+ """The pipeline-authoritative outcome stored in the manifest (extra key)."""
505
+ extra = manifest.model_extra or {}
506
+ return str(extra.get("outcome") or "").strip().upper()
507
+
508
+
509
+ def _no_knowledge_evidence_valid(
510
+ manifest: ManifestV2, entries: Sequence[OutputEntry | None]
511
+ ) -> bool:
512
+ """Evidence/policy check for reusing a NO_KNOWLEDGE verdict (fail closed)."""
513
+ for entry in entries:
514
+ if entry is None:
515
+ return False
516
+ if entry.evidence_fingerprint and manifest.source_revision:
517
+ # When both sides recorded evidence it must agree. A manifest that
518
+ # predates the evidence field cannot disprove the verdict; the
519
+ # reason/validator checks below still apply.
520
+ if entry.evidence_fingerprint != manifest.source_revision:
521
+ return False
522
+ if entry.validator_version and entry.validator_version != OUTPUT_VALIDATOR_VERSION:
523
+ return False
524
+ return True
525
+
526
+
527
+ def derive_overall_status(
528
+ manifest: ManifestV2, *, requested: Sequence[str] | None = None
529
+ ) -> tuple[str, list[str]]:
530
+ """Map a manifest to ``(overall_status, reasons)`` per plan §4.5.
531
+
532
+ ``unknown`` (stage or unresolved remote submission) maps to ``blocked`` with
533
+ a reason code — it is never reported as complete.
534
+ """
535
+ requested_formats = list(dict.fromkeys(requested or list(manifest.outputs)))
536
+ entries = {fmt: manifest.outputs.get(fmt) for fmt in requested_formats}
537
+ reasons: list[str] = []
538
+
539
+ unresolved = unresolved_remote_tasks(manifest)
540
+ for provider, task in unresolved.items():
541
+ status = str(task.get("status") or "unknown")
542
+ reasons.append(f"{REASON_REMOTE_SUBMISSION_UNKNOWN}:{provider}:{status}")
543
+ unknown_stages = sorted(stage for stage, status in manifest.stages.items() if status == "unknown")
544
+ for stage in unknown_stages:
545
+ reasons.append(f"{REASON_UNKNOWN_STAGE}:{stage}")
546
+ if unresolved or unknown_stages:
547
+ return OVERALL_BLOCKED, reasons
548
+
549
+ # NO_KNOWLEDGE is a content verdict and takes precedence over a format-level
550
+ # not-applicable judgment (plan §4.5 last paragraph, R-05/R-11). It is
551
+ # honoured from the pipeline-recorded ``outcome`` extra key OR the per-format
552
+ # ``reason_code``/``reason`` — but only while the recorded evidence and the
553
+ # judgment-policy version still match, otherwise the source is re-judged.
554
+ if requested_formats and all(
555
+ entry is not None and entry.status == "not_applicable" for entry in entries.values()
556
+ ):
557
+ not_applicable_entries = [entries[fmt] for fmt in requested_formats]
558
+ claims_no_knowledge = manifest_recorded_outcome(manifest) == OVERALL_NO_KNOWLEDGE or all(
559
+ entry is not None and output_entry_claims_no_knowledge(entry)
560
+ for entry in not_applicable_entries
561
+ )
562
+ if claims_no_knowledge and _no_knowledge_evidence_valid(
563
+ manifest, not_applicable_entries
564
+ ):
565
+ return OVERALL_NO_KNOWLEDGE, [
566
+ f"{REASON_NO_KNOWLEDGE}: no knowledge atoms for complete content"
567
+ ]
568
+ reasons.append(
569
+ f"{REASON_OUTPUTS_NOT_APPLICABLE}: every requested format is not applicable"
570
+ )
571
+ return OVERALL_FAILED, reasons
572
+
573
+ failed = [
574
+ fmt
575
+ for fmt, entry in entries.items()
576
+ if entry is not None and entry.status == "failed"
577
+ ]
578
+ blocked = [
579
+ fmt
580
+ for fmt, entry in entries.items()
581
+ if entry is not None and entry.status == "blocked"
582
+ ]
583
+ failed_stages = sorted(stage for stage, status in manifest.stages.items() if status == "failed")
584
+ blocked_stages = sorted(stage for stage, status in manifest.stages.items() if status == "blocked")
585
+ if failed or failed_stages:
586
+ reasons.extend(f"{REASON_ARTIFACT_MISSING}:{fmt}" for fmt in failed)
587
+ reasons.extend(f"STAGE_FAILED:{stage}" for stage in failed_stages)
588
+ return OVERALL_FAILED, reasons
589
+ if blocked or blocked_stages:
590
+ reasons.extend(f"OUTPUT_BLOCKED:{fmt}" for fmt in blocked)
591
+ reasons.extend(f"STAGE_BLOCKED:{stage}" for stage in blocked_stages)
592
+ return OVERALL_BLOCKED, reasons
593
+
594
+ partial = (
595
+ [fmt for fmt, entry in entries.items() if entry is not None and entry.status == "partial"]
596
+ + [fmt for fmt, entry in entries.items() if entry is not None and entry.status == "not_applicable"]
597
+ + [fmt for fmt, entry in entries.items() if entry is not None and entry.status in {"pending", "skipped"}]
598
+ + [fmt for fmt in requested_formats if entries.get(fmt) is None]
599
+ + [stage for stage, status in manifest.stages.items() if status in {"pending", "running", "partial"}]
600
+ )
601
+ if partial:
602
+ reasons.extend(f"INCOMPLETE:{item}" for item in partial)
603
+ return OVERALL_PARTIAL, reasons
604
+
605
+ if manifest.missing:
606
+ reasons.extend(f"MISSING:{item}" for item in manifest.missing)
607
+ return OVERALL_PARTIAL, reasons
608
+
609
+ return OVERALL_COMPLETE, []
610
+
611
+
612
+ # --------------------------------------------------------------------------- #
613
+ # Artifact store
614
+ # --------------------------------------------------------------------------- #
615
+ class ArtifactStore:
616
+ """Owns locked, atomic writes for one task output directory."""
617
+
618
+ def __init__(self, root: Path | str, *, lock_timeout: float = 10.0) -> None:
619
+ self.root = Path(root)
620
+ self.lock_timeout = float(lock_timeout)
621
+ self._lock = _ReentrantPathLock.for_path(self.root / f".ytk-artifacts{_LOCK_SUFFIX}")
622
+
623
+ # ------------------------------------------------------------- properties
624
+ @property
625
+ def lock_path(self) -> Path:
626
+ return self._lock.path
627
+
628
+ @property
629
+ def manifest_path(self) -> Path:
630
+ return self.root / MANIFEST_FILENAME
631
+
632
+ # ---------------------------------------------------------------- locking
633
+ @contextmanager
634
+ def locked(self, timeout: float | None = None) -> Iterator[None]:
635
+ with self._lock.acquire(self.lock_timeout if timeout is None else timeout):
636
+ yield
637
+
638
+ # ------------------------------------------------------------- path safety
639
+ def _safe_relpath(self, relpath: str | Path) -> Path:
640
+ candidate = Path(relpath)
641
+ if candidate.is_absolute() or candidate.drive:
642
+ raise ArtifactStoreError(f"artifact path must be relative: {relpath!r}")
643
+ if not candidate.parts or any(part == ".." for part in candidate.parts):
644
+ raise ArtifactStoreError(f"artifact path escapes the task directory: {relpath!r}")
645
+ return self.root / candidate
646
+
647
+ # ------------------------------------------------------------------ write
648
+ def write_bytes(
649
+ self, relpath: str | Path, data: bytes, *, sha256: str | None = None
650
+ ) -> Path:
651
+ """Validate then atomically write raw bytes; returns the written path."""
652
+ if not isinstance(data, (bytes, bytearray, memoryview)):
653
+ raise TypeError("artifact payload must be bytes")
654
+ payload = bytes(data)
655
+ digest = sha256_bytes(payload)
656
+ if sha256 is not None and sha256 != digest:
657
+ raise ArtifactStoreError(
658
+ f"refusing to write {relpath}: declared sha256 {sha256} != computed {digest}"
659
+ )
660
+ path = self._safe_relpath(relpath)
661
+ with self._lock.acquire(self.lock_timeout):
662
+ atomic_write_bytes(path, payload)
663
+ return path
664
+
665
+ def write_json(self, relpath: str | Path, payload: object) -> Path:
666
+ """Serialise, validate the round-trip, then atomically replace the file."""
667
+ data = _dump_json(payload)
668
+ try:
669
+ json.loads(data)
670
+ except json.JSONDecodeError as exc: # pragma: no cover - defensive
671
+ raise ArtifactStoreError(f"refusing to write invalid JSON to {relpath}: {exc}") from exc
672
+ path = self._safe_relpath(relpath)
673
+ with self._lock.acquire(self.lock_timeout):
674
+ atomic_write_bytes(path, data)
675
+ return path
676
+
677
+ # ------------------------------------------------------------------- read
678
+ def read_json(self, relpath: str | Path) -> object | None:
679
+ path = self._safe_relpath(relpath)
680
+ try:
681
+ return json.loads(path.read_text(encoding="utf-8"))
682
+ except FileNotFoundError:
683
+ return None
684
+ except (OSError, json.JSONDecodeError) as exc:
685
+ LOGGER.warning("artifact %s unreadable (%s): treating as missing", path, exc)
686
+ return None
687
+
688
+ def _read_manifest_unlocked(self) -> ManifestV2 | None:
689
+ try:
690
+ text = self.manifest_path.read_text(encoding="utf-8")
691
+ except FileNotFoundError:
692
+ return None
693
+ except OSError as exc:
694
+ LOGGER.warning("manifest %s unreadable (%s): treating as missing", self.manifest_path, exc)
695
+ return None
696
+ try:
697
+ payload = json.loads(text)
698
+ except json.JSONDecodeError as exc:
699
+ LOGGER.warning("manifest %s corrupt (%s): treating as missing", self.manifest_path, exc)
700
+ return None
701
+ if not isinstance(payload, dict):
702
+ return None
703
+ try:
704
+ manifest = ManifestV2.model_validate(payload)
705
+ except ValidationError as exc:
706
+ LOGGER.warning("manifest %s invalid (%s): treating as missing", self.manifest_path, exc)
707
+ return None
708
+ if manifest.schema_version != MANIFEST_SCHEMA_VERSION:
709
+ return None
710
+ return manifest
711
+
712
+ def read_manifest(self) -> ManifestV2 | None:
713
+ """Return the manifest, or ``None`` when missing/corrupt (never raises)."""
714
+ return self._read_manifest_unlocked()
715
+
716
+ # ---------------------------------------------------------- manifest write
717
+ def write_manifest(self, manifest: ManifestV2) -> Path:
718
+ """Write the manifest **last**, after validating every claimed output."""
719
+ if not isinstance(manifest, ManifestV2):
720
+ raise TypeError("write_manifest expects a ManifestV2")
721
+ with self._lock.acquire(self.lock_timeout):
722
+ existing = self._read_manifest_unlocked()
723
+ assert_manifest_ownership(existing, manifest)
724
+ prepared = self._prepare_manifest_for_write(manifest)
725
+ path = self.manifest_path
726
+ atomic_write_bytes(path, _dump_json(prepared))
727
+ return path
728
+
729
+ def _prepare_manifest_for_write(self, manifest: ManifestV2) -> ManifestV2:
730
+ prepared = manifest.model_copy(deep=True)
731
+ prepared.updated_at = prepared.updated_at or utc_now()
732
+ problems: list[str] = []
733
+ for fmt, entry in prepared.outputs.items():
734
+ if entry.status != "complete":
735
+ continue
736
+ target = self._output_path(entry.path)
737
+ if target is None or not target.exists():
738
+ problems.append(f"{fmt}: missing output file {entry.path!r}")
739
+ continue
740
+ if _entry_is_directory_artifact(entry, target):
741
+ if not target.is_dir():
742
+ problems.append(f"{fmt}: missing output directory {entry.path!r}")
743
+ continue
744
+ file_count = directory_file_count(target)
745
+ if file_count <= 0:
746
+ problems.append(f"{fmt}: empty output directory {entry.path!r}")
747
+ continue
748
+ recorded_count = _entry_extra(entry).get("file_count")
749
+ if recorded_count is not None:
750
+ try:
751
+ expected = int(recorded_count)
752
+ except (TypeError, ValueError):
753
+ problems.append(f"{fmt}: invalid file_count for {entry.path!r}")
754
+ continue
755
+ if expected != file_count:
756
+ problems.append(
757
+ f"{fmt}: file_count mismatch for {entry.path!r} "
758
+ f"({expected} != {file_count})"
759
+ )
760
+ continue
761
+ tree_digest = directory_digest(target)
762
+ recorded_digest = _entry_extra(entry).get("tree_sha256")
763
+ if tree_digest is None or (
764
+ recorded_digest is not None and recorded_digest != tree_digest
765
+ ):
766
+ problems.append(f"{fmt}: directory digest mismatch for {entry.path!r}")
767
+ continue
768
+ entry.tree_sha256 = tree_digest # type: ignore[attr-defined]
769
+ entry.sha256 = None
770
+ if entry.path is None:
771
+ entry.path = (
772
+ str(target.relative_to(self.root))
773
+ if self._is_relative_to(target)
774
+ else str(target)
775
+ )
776
+ try:
777
+ entry.file_count = file_count # type: ignore[attr-defined]
778
+ except (AttributeError, ValueError): # pragma: no cover - pydantic extra
779
+ pass
780
+ continue
781
+ digest = sha256_file(target)
782
+ if entry.sha256 and entry.sha256 != digest:
783
+ problems.append(f"{fmt}: sha256 mismatch for {entry.path!r}")
784
+ continue
785
+ entry.sha256 = digest
786
+ if entry.path is None:
787
+ entry.path = str(target.relative_to(self.root)) if self._is_relative_to(target) else str(target)
788
+ if problems:
789
+ raise ArtifactStoreError(
790
+ "refusing to write a manifest that claims completion for unverifiable outputs: "
791
+ + "; ".join(problems)
792
+ )
793
+ return prepared
794
+
795
+ def _output_path(self, recorded: str | None) -> Path | None:
796
+ if not recorded:
797
+ return None
798
+ candidate = Path(recorded)
799
+ target = candidate if candidate.is_absolute() else self.root / candidate
800
+ if not self._is_relative_to(target) or target.is_symlink():
801
+ return None
802
+ return target
803
+
804
+ def _is_relative_to(self, target: Path) -> bool:
805
+ try:
806
+ target.resolve().relative_to(self.root.resolve())
807
+ except (OSError, ValueError):
808
+ return False
809
+ return True
810
+
811
+ # --------------------------------------------------------------- validity
812
+ def output_valid(
813
+ self, fmt: str, *, validator_version: str, check_digest: bool = True
814
+ ) -> bool:
815
+ """True only when the manifest *and* the actual file both verify.
816
+
817
+ Missing manifest entry, non-complete status, validator mismatch, missing
818
+ file, or digest mismatch all return ``False`` (fail closed — a corrupt
819
+ artifact can never be used to skip work).
820
+ """
821
+ manifest = self.read_manifest()
822
+ if manifest is None:
823
+ return False
824
+ entry = manifest.outputs.get(fmt)
825
+ if entry is None or entry.status != "complete":
826
+ return False
827
+ if not entry.validator_version or entry.validator_version != validator_version:
828
+ return False
829
+ target = self._output_path(entry.path)
830
+ if target is None or not target.exists():
831
+ return False
832
+ if _entry_is_directory_artifact(entry, target):
833
+ if not target.is_dir():
834
+ return False
835
+ file_count = directory_file_count(target)
836
+ if file_count <= 0:
837
+ return False
838
+ recorded_count = _entry_extra(entry).get("file_count")
839
+ if recorded_count is not None:
840
+ try:
841
+ if int(recorded_count) != file_count:
842
+ return False
843
+ except (TypeError, ValueError):
844
+ return False
845
+ recorded_digest = _entry_extra(entry).get("tree_sha256")
846
+ if not recorded_digest:
847
+ return False
848
+ if check_digest and directory_digest(target) != recorded_digest:
849
+ return False
850
+ return True
851
+ if not target.is_file():
852
+ return False
853
+ if not entry.sha256:
854
+ return False
855
+ if check_digest:
856
+ try:
857
+ if sha256_file(target) != entry.sha256:
858
+ return False
859
+ except OSError:
860
+ return False
861
+ return True
862
+
863
+ # ---------------------------------------------------------------- resume
864
+ def should_reuse_output(
865
+ self,
866
+ fmt: str,
867
+ *,
868
+ request_fingerprint: str,
869
+ validator_version: str,
870
+ source_revision: str | None = None,
871
+ refresh_epoch: int | None = None,
872
+ check_digest: bool = True,
873
+ ) -> bool:
874
+ """Decide whether an already-rendered format can be reused as-is."""
875
+ manifest = self.read_manifest()
876
+ if manifest is None:
877
+ return False
878
+ if not request_fingerprint or manifest.request_fingerprint != request_fingerprint:
879
+ return False
880
+ if source_revision is not None and manifest.source_revision not in (None, "") and (
881
+ manifest.source_revision != source_revision
882
+ ):
883
+ return False
884
+ if refresh_epoch is not None and manifest.refresh_epoch != refresh_epoch:
885
+ return False
886
+ return self.output_valid(fmt, validator_version=validator_version, check_digest=check_digest)
887
+
888
+ def should_skip_format(
889
+ self,
890
+ fmt: str,
891
+ *,
892
+ request_fingerprint: str | None = None,
893
+ evidence_fingerprint: str | None = None,
894
+ checker_version: str | None = None,
895
+ validator_version: str | None = None,
896
+ source_revision: str | None = None,
897
+ ) -> tuple[bool, str | None]:
898
+ """Return ``(skip, reason_code)`` for one requested format.
899
+
900
+ * a verified complete output is skipped (reason ``None``);
901
+ * ``not_applicable`` is only skipped while its justification still holds
902
+ (same evidence fingerprint **and** checker version) — a changed
903
+ judgment policy or evidence forces a re-evaluation;
904
+ * blocked/unknown states are never skipped.
905
+ """
906
+ manifest = self.read_manifest()
907
+ if manifest is None:
908
+ return (False, None)
909
+ entry = manifest.outputs.get(fmt)
910
+ if entry is None:
911
+ return (False, None)
912
+ if entry.status == "complete":
913
+ if validator_version is None:
914
+ return (False, None)
915
+ if request_fingerprint is not None and manifest.request_fingerprint != request_fingerprint:
916
+ return (False, None)
917
+ if self.should_reuse_output(
918
+ fmt,
919
+ request_fingerprint=manifest.request_fingerprint,
920
+ validator_version=validator_version,
921
+ source_revision=source_revision,
922
+ ):
923
+ return (True, None)
924
+ return (False, None)
925
+ if entry.status == "not_applicable":
926
+ evidence_ok = bool(evidence_fingerprint) and (
927
+ entry.evidence_fingerprint == evidence_fingerprint
928
+ )
929
+ checker_ok = bool(checker_version) and (
930
+ entry.checker_version == checker_version
931
+ )
932
+ policy_ok = bool(validator_version) and (
933
+ entry.validator_version == validator_version
934
+ )
935
+ if output_entry_claims_no_knowledge(entry) and evidence_ok and (
936
+ checker_ok or policy_ok
937
+ ):
938
+ # NO_KNOWLEDGE is a content verdict, not an "empty skill" failure:
939
+ # reuse it while the evidence and judgment policy version match.
940
+ return (True, entry.reason_code or REASON_NO_KNOWLEDGE)
941
+ if not evidence_ok or not checker_ok:
942
+ return (False, None) # cannot prove the judgment still holds
943
+ return (True, entry.reason_code or "NOT_APPLICABLE")
944
+ return (False, None)
945
+
946
+ # ----------------------------------------------------------- remote state
947
+ def unresolved_remote_tasks(self, provider: str | None = None) -> dict[str, dict[str, Any]]:
948
+ manifest = self.read_manifest()
949
+ if manifest is None:
950
+ return {}
951
+ return unresolved_remote_tasks(manifest, provider)
952
+
953
+ def has_unresolved_remote_submission(self, provider: str | None = None) -> bool:
954
+ return bool(self.unresolved_remote_tasks(provider))
955
+
956
+ def record_remote_task(
957
+ self,
958
+ provider: str,
959
+ *,
960
+ task_id: str | None = None,
961
+ status: str = "submitted",
962
+ deadline: float | None = None,
963
+ submitted_at: str | None = None,
964
+ **extra: Any,
965
+ ) -> ManifestV2:
966
+ """Insert/update one provider's remote task record (never deletes others)."""
967
+ with self._lock.acquire(self.lock_timeout):
968
+ manifest = self._read_manifest_unlocked() or new_manifest(
969
+ source_ids=[], request_fingerprint=""
970
+ )
971
+ updated = manifest.model_copy(deep=True)
972
+ record: dict[str, Any] = dict(updated.remote_tasks.get(provider) or {})
973
+ record.update(extra)
974
+ record["task_id"] = task_id
975
+ record["status"] = status
976
+ record["deadline"] = deadline
977
+ record["submitted_at"] = submitted_at or record.get("submitted_at")
978
+ updated.remote_tasks[provider] = record
979
+ updated.updated_at = utc_now()
980
+ atomic_write_bytes(self.manifest_path, _dump_json(updated))
981
+ return updated
982
+
983
+ # -------------------------------------------------------------- refresh
984
+ def invalidate_for_refresh(
985
+ self,
986
+ *,
987
+ refresh_epoch: int | None = None,
988
+ source_revision: str | None = None,
989
+ invalidate_stages: Sequence[str] = (),
990
+ invalidate_formats: Sequence[str] = (),
991
+ reason: str = REASON_REFRESH,
992
+ ) -> ManifestV2:
993
+ """Invalidate only what a refresh actually changed (plan §4.6/§9.1).
994
+
995
+ The request fingerprint is **not** rewritten here and remote-task records
996
+ are never deleted. When an unresolved remote submission exists, a
997
+ refresh that would invalidate acquisition-dependent stages is refused —
998
+ a new fingerprint must never be used to forget a possibly-running job.
999
+ """
1000
+ with self._lock.acquire(self.lock_timeout):
1001
+ manifest = self._read_manifest_unlocked()
1002
+ if manifest is None:
1003
+ raise ArtifactStoreError(
1004
+ f"cannot refresh {self.root}: no valid manifest to invalidate"
1005
+ )
1006
+ unresolved = unresolved_remote_tasks(manifest)
1007
+ stages = set(invalidate_stages)
1008
+ if unresolved and (not stages or stages & _ACQUISITION_STAGES):
1009
+ providers = ", ".join(sorted(unresolved))
1010
+ raise UnresolvedRemoteSubmissionError(
1011
+ "refusing to invalidate acquisition state: remote submission(s) "
1012
+ f"still unresolved for {providers} — resolve or poll the existing "
1013
+ "task first, never re-submit it behind a new fingerprint"
1014
+ )
1015
+ updated = manifest.model_copy(deep=True)
1016
+ updated.refresh_epoch = (
1017
+ int(refresh_epoch) if refresh_epoch is not None else manifest.refresh_epoch + 1
1018
+ )
1019
+ if source_revision is not None:
1020
+ updated.source_revision = source_revision
1021
+ for stage in stages:
1022
+ updated.stages[stage] = "pending"
1023
+ for fmt in invalidate_formats:
1024
+ entry = updated.outputs.get(fmt) or OutputEntry()
1025
+ entry.status = "pending"
1026
+ entry.sha256 = None
1027
+ entry.validator_version = None
1028
+ entry.reason = reason
1029
+ entry.reason_code = reason
1030
+ entry.updated_at = utc_now()
1031
+ updated.outputs[fmt] = entry
1032
+ updated.updated_at = utc_now()
1033
+ # Remote-task records (including unknown ones) are copied verbatim.
1034
+ atomic_write_bytes(self.manifest_path, _dump_json(updated))
1035
+ return updated
1036
+
1037
+ # ------------------------------------------------------------- ownership
1038
+ def assert_owned_by(self, source_ids: Sequence[str]) -> None:
1039
+ """Raise when an existing manifest belongs to a different work."""
1040
+ assert_manifest_ownership(self._read_manifest_unlocked(), source_ids)
1041
+
1042
+
1043
+ def assert_manifest_ownership(
1044
+ existing: ManifestV2 | None, incoming: ManifestV2 | Sequence[str]
1045
+ ) -> None:
1046
+ """Refuse to share one output directory between different works."""
1047
+ if existing is None or not existing.source_ids:
1048
+ return
1049
+ incoming_ids = (
1050
+ list(incoming.source_ids) if isinstance(incoming, ManifestV2) else list(incoming)
1051
+ )
1052
+ if not incoming_ids:
1053
+ return
1054
+ if not set(existing.source_ids) & set(incoming_ids):
1055
+ raise ArtifactOwnershipError(
1056
+ "output directory is owned by a different source: "
1057
+ f"existing={sorted(existing.source_ids)!r} incoming={sorted(incoming_ids)!r}; "
1058
+ "use a separate --out directory for each work"
1059
+ )
1060
+
1061
+
1062
+ __all__ = [
1063
+ "ArtifactLockTimeoutError",
1064
+ "ArtifactOwnershipError",
1065
+ "ArtifactStore",
1066
+ "ArtifactStoreError",
1067
+ "MANIFEST_FILENAME",
1068
+ "MANIFEST_SCHEMA_VERSION",
1069
+ "ManifestV2",
1070
+ "OVERALL_BLOCKED",
1071
+ "OVERALL_COMPLETE",
1072
+ "OVERALL_FAILED",
1073
+ "OVERALL_NO_KNOWLEDGE",
1074
+ "OVERALL_PARTIAL",
1075
+ "OUTPUT_VALIDATOR_VERSION",
1076
+ "OutputEntry",
1077
+ "OutputStatus",
1078
+ "REASON_ARTIFACT_CORRUPT",
1079
+ "REASON_ARTIFACT_MISSING",
1080
+ "REASON_DISPATCHING_UNKNOWN",
1081
+ "REASON_NO_KNOWLEDGE",
1082
+ "REASON_OUTPUTS_NOT_APPLICABLE",
1083
+ "REASON_REFRESH",
1084
+ "REASON_REMOTE_SUBMISSION_UNKNOWN",
1085
+ "REASON_UNKNOWN_STAGE",
1086
+ "REASON_VALIDATOR_MISMATCH",
1087
+ "RESOLVED_REMOTE_STATUSES",
1088
+ "StageName",
1089
+ "StageStatus",
1090
+ "UNRESOLVED_REMOTE_STATUSES",
1091
+ "UnresolvedRemoteSubmissionError",
1092
+ "assert_manifest_ownership",
1093
+ "atomic_write_bytes",
1094
+ "derive_overall_status",
1095
+ "directory_digest",
1096
+ "directory_file_count",
1097
+ "mark_stage",
1098
+ "manifest_recorded_outcome",
1099
+ "new_manifest",
1100
+ "output_entry_claims_no_knowledge",
1101
+ "record_not_applicable",
1102
+ "record_output",
1103
+ "sha256_bytes",
1104
+ "sha256_file",
1105
+ "unresolved_remote_tasks",
1106
+ "utc_now",
1107
+ ]