kd-engine 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kd_engine-1.0.0.dist-info/METADATA +19 -0
- kd_engine-1.0.0.dist-info/RECORD +118 -0
- kd_engine-1.0.0.dist-info/WHEEL +5 -0
- kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
- kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
- kd_engine-1.0.0.dist-info/top_level.txt +1 -0
- ytk/__init__.py +2 -0
- ytk/__main__.py +21 -0
- ytk/batch/__init__.py +39 -0
- ytk/batch/runner.py +1149 -0
- ytk/cache/__init__.py +34 -0
- ytk/cache/v2.py +562 -0
- ytk/cleaning/density.py +40 -0
- ytk/cleaning/fillers.py +18 -0
- ytk/cleaning/sentence_boundary.py +30 -0
- ytk/cleaning/service.py +62 -0
- ytk/cli.py +1281 -0
- ytk/commands/__init__.py +3 -0
- ytk/commands/merge.py +600 -0
- ytk/config.py +757 -0
- ytk/dedup/service.py +166 -0
- ytk/doctor.py +891 -0
- ytk/dotenv.py +334 -0
- ytk/errors.py +38 -0
- ytk/extraction/service.py +451 -0
- ytk/extraction/strategies/base.py +16 -0
- ytk/extraction/strategies/code_block.py +285 -0
- ytk/extraction/strategies/heuristic.py +791 -0
- ytk/extraction/strategies/local_model.py +56 -0
- ytk/extraction/text_context.py +177 -0
- ytk/extraction/visual_context.py +159 -0
- ytk/ingestion/adapters/__init__.py +1 -0
- ytk/ingestion/adapters/_common.py +865 -0
- ytk/ingestion/adapters/base.py +160 -0
- ytk/ingestion/adapters/bilibili.py +271 -0
- ytk/ingestion/adapters/douyin.py +445 -0
- ytk/ingestion/adapters/tiktok.py +102 -0
- ytk/ingestion/adapters/xiaohongshu.py +758 -0
- ytk/ingestion/adapters/youtube.py +74 -0
- ytk/ingestion/browser_acquired.py +220 -0
- ytk/ingestion/http_client.py +43 -0
- ytk/ingestion/local_import.py +202 -0
- ytk/ingestion/media.py +468 -0
- ytk/ingestion/providers/asr.py +902 -0
- ytk/ingestion/providers/base.py +13 -0
- ytk/ingestion/providers/dtk_client.py +816 -0
- ytk/ingestion/providers/minimax.py +350 -0
- ytk/ingestion/providers/transcript_api.py +336 -0
- ytk/ingestion/providers/xhs_client.py +646 -0
- ytk/ingestion/providers/ytdlp.py +400 -0
- ytk/ingestion/registry.py +189 -0
- ytk/ingestion/resolver.py +417 -0
- ytk/ingestion/search.py +56 -0
- ytk/ingestion/service.py +167 -0
- ytk/ingestion/subtitles.py +602 -0
- ytk/ingestion/transcript_cache.py +212 -0
- ytk/ingestion/url_parser.py +53 -0
- ytk/ingestion/video_frames.py +292 -0
- ytk/ingestion/ytdlp_acquire.py +754 -0
- ytk/logging.py +383 -0
- ytk/models/common.py +20 -0
- ytk/models/content.py +451 -0
- ytk/models/knowledge.py +43 -0
- ytk/models/migrations.py +262 -0
- ytk/models/skill.py +13 -0
- ytk/models/source.py +265 -0
- ytk/models/topic.py +18 -0
- ytk/models/transcript.py +30 -0
- ytk/ocr/__init__.py +65 -0
- ytk/ocr/engine.py +245 -0
- ytk/ocr/layout.py +347 -0
- ytk/ocr/pipeline.py +440 -0
- ytk/ocr/rapidocr_engine.py +144 -0
- ytk/pipeline/acquisition_checkpoint.py +240 -0
- ytk/pipeline/artifacts.py +18 -0
- ytk/pipeline/orchestrator.py +598 -0
- ytk/pipeline/processing_identity.py +18 -0
- ytk/pipeline/retry.py +728 -0
- ytk/pipeline/runners.py +578 -0
- ytk/pipeline/state.py +1107 -0
- ytk/pipeline/v2.py +1863 -0
- ytk/py.typed +1 -0
- ytk/quality/__init__.py +9 -0
- ytk/quality/completeness.py +260 -0
- ytk/renderers/base.py +9 -0
- ytk/renderers/citation_alignment.py +617 -0
- ytk/renderers/evidence_formatter.py +362 -0
- ytk/renderers/json_renderer.py +13 -0
- ytk/renderers/jsonl_renderer.py +18 -0
- ytk/renderers/knowledge_doc_renderer.py +1984 -0
- ytk/renderers/markdown_renderer.py +540 -0
- ytk/renderers/obsidian_renderer.py +138 -0
- ytk/renderers/skill_package.py +21 -0
- ytk/renderers/source_links.py +309 -0
- ytk/renderers/summary_renderer.py +163 -0
- ytk/renderers/v2.py +171 -0
- ytk/renderers/v2_jsonl.py +165 -0
- ytk/renderers/v2_markdown.py +477 -0
- ytk/renderers/v2_obsidian.py +113 -0
- ytk/renderers/v2_summary.py +220 -0
- ytk/segmentation/change_points.py +50 -0
- ytk/segmentation/embeddings.py +45 -0
- ytk/segmentation/service.py +318 -0
- ytk/skill_builder/__init__.py +1 -0
- ytk/skill_builder/builder.py +419 -0
- ytk/skill_builder/templates/SKILL.md.j2 +85 -0
- ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
- ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
- ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
- ytk/skill_builder/templates/references/commands.md.j2 +34 -0
- ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
- ytk/skill_builder/templates/references/configs.md.j2 +31 -0
- ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
- ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
- ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
- ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
- ytk/skill_builder/templates.py +29 -0
- ytk/text_normalization.py +44 -0
ytk/pipeline/state.py
ADDED
|
@@ -0,0 +1,1107 @@
|
|
|
1
|
+
"""T12.2 — v2 artifact manifest, atomic writes and restart decisions.
|
|
2
|
+
|
|
3
|
+
Frozen by ``docs/implementation/five-platform/contracts-v2.md`` §1.4:
|
|
4
|
+
|
|
5
|
+
* :class:`ManifestV2` records source identity, the acquisition revision, the
|
|
6
|
+
request fingerprint, per-stage status, per-format outputs, missing items,
|
|
7
|
+
remote tasks and the refresh epoch. Extra keys are accepted so the v2
|
|
8
|
+
pipeline can keep its ``outcome``/``reasons``/``refresh`` diagnostics in the
|
|
9
|
+
same file (the v2 pipeline treats ``outcome`` as authoritative and the stage
|
|
10
|
+
map as diagnostic).
|
|
11
|
+
* :class:`ArtifactStore` owns every write into one task directory: locking
|
|
12
|
+
(``filelock`` when installed, stdlib ``fcntl.flock`` fallback, per-path
|
|
13
|
+
reentrant), temp-file + validate + ``os.replace`` atomic writes, and the
|
|
14
|
+
manifest **last**.
|
|
15
|
+
* A corrupt/missing output or manifest can never be used to resume: digest and
|
|
16
|
+
validator-version mismatches fail closed. A task directory owned by another
|
|
17
|
+
source is refused (different works must not share ``out``).
|
|
18
|
+
* ``unknown`` only ever maps to an overall ``blocked`` outcome with a reason
|
|
19
|
+
code; an unresolved remote submission (DISPATCHING / unknown task id) can
|
|
20
|
+
never be bypassed by changing fingerprints or refreshing.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import hashlib
|
|
26
|
+
import json
|
|
27
|
+
import logging
|
|
28
|
+
import os
|
|
29
|
+
import tempfile
|
|
30
|
+
import threading
|
|
31
|
+
import time
|
|
32
|
+
from collections.abc import Iterator, Mapping, Sequence
|
|
33
|
+
from contextlib import contextmanager
|
|
34
|
+
from datetime import UTC, datetime
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
from typing import Any, Literal
|
|
37
|
+
|
|
38
|
+
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
|
39
|
+
|
|
40
|
+
LOGGER = logging.getLogger(__name__)
|
|
41
|
+
|
|
42
|
+
MANIFEST_FILENAME = "manifest.json"
|
|
43
|
+
MANIFEST_SCHEMA_VERSION = 2
|
|
44
|
+
|
|
45
|
+
StageName = Literal[
|
|
46
|
+
"resolve",
|
|
47
|
+
"acquire",
|
|
48
|
+
"subtitle",
|
|
49
|
+
"media",
|
|
50
|
+
"asr",
|
|
51
|
+
"ocr",
|
|
52
|
+
"segment",
|
|
53
|
+
"extract",
|
|
54
|
+
"render",
|
|
55
|
+
]
|
|
56
|
+
StageStatus = Literal["pending", "running", "complete", "partial", "blocked", "failed", "unknown"]
|
|
57
|
+
OutputStatus = Literal[
|
|
58
|
+
"pending", "complete", "partial", "failed", "blocked", "not_applicable", "skipped"
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
#: Overall outcomes (plan §4.5 / contract §1.4).
|
|
62
|
+
OVERALL_COMPLETE = "COMPLETE"
|
|
63
|
+
OVERALL_PARTIAL = "PARTIAL"
|
|
64
|
+
OVERALL_BLOCKED = "BLOCKED"
|
|
65
|
+
OVERALL_FAILED = "FAILED"
|
|
66
|
+
OVERALL_NO_KNOWLEDGE = "NO_KNOWLEDGE"
|
|
67
|
+
|
|
68
|
+
#: Reason codes used by the resume/refresh machinery and the CLI exit mapping.
|
|
69
|
+
REASON_REMOTE_SUBMISSION_UNKNOWN = "REMOTE_SUBMISSION_UNKNOWN"
|
|
70
|
+
REASON_DISPATCHING_UNKNOWN = "DISPATCHING_UNKNOWN"
|
|
71
|
+
REASON_UNKNOWN_STAGE = "UNKNOWN_STAGE"
|
|
72
|
+
REASON_OUTPUTS_NOT_APPLICABLE = "OUTPUTS_NOT_APPLICABLE"
|
|
73
|
+
REASON_NO_KNOWLEDGE = "NO_KNOWLEDGE"
|
|
74
|
+
REASON_REFRESH = "REFRESH"
|
|
75
|
+
REASON_ARTIFACT_CORRUPT = "ARTIFACT_CORRUPT"
|
|
76
|
+
REASON_ARTIFACT_MISSING = "ARTIFACT_MISSING"
|
|
77
|
+
REASON_VALIDATOR_MISMATCH = "VALIDATOR_MISMATCH"
|
|
78
|
+
|
|
79
|
+
#: Output validator version written by the v2 pipeline. It doubles as the
|
|
80
|
+
#: judgment-policy version when an entry records ``validator_version`` but no
|
|
81
|
+
#: explicit ``checker_version`` (R-05/R-11).
|
|
82
|
+
OUTPUT_VALIDATOR_VERSION = "v2-1"
|
|
83
|
+
|
|
84
|
+
#: Remote submission states that are *not resolved* yet. A task in one of
|
|
85
|
+
#: these states must never be re-submitted or hidden behind a new fingerprint.
|
|
86
|
+
UNRESOLVED_REMOTE_STATUSES: frozenset[str] = frozenset(
|
|
87
|
+
{"dispatching", "unknown", "pending", "running", "in_progress", "queued", "submitted"}
|
|
88
|
+
)
|
|
89
|
+
#: Remote states that *are* resolved (success or a terminal failure).
|
|
90
|
+
RESOLVED_REMOTE_STATUSES: frozenset[str] = frozenset(
|
|
91
|
+
{"complete", "completed", "done", "succeeded", "failed", "cancelled", "canceled",
|
|
92
|
+
"expired", "resolved", "not_required"}
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
#: Stages that depend on acquisition/remote work (privileged for refresh).
|
|
96
|
+
_ACQUISITION_STAGES: frozenset[str] = frozenset(
|
|
97
|
+
{"resolve", "acquire", "subtitle", "media", "asr", "ocr"}
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
_LOCK_SUFFIX = ".lock"
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
# --------------------------------------------------------------------------- #
|
|
104
|
+
# Locking: filelock → fcntl fallback → process-local
|
|
105
|
+
# --------------------------------------------------------------------------- #
|
|
106
|
+
try: # optional dependency (``lock`` extra); never required by a base install
|
|
107
|
+
import filelock as _filelock
|
|
108
|
+
|
|
109
|
+
_FILELOCK_AVAILABLE = True
|
|
110
|
+
except ImportError: # pragma: no cover - exercised by the fallback test
|
|
111
|
+
_filelock = None # type: ignore[assignment]
|
|
112
|
+
_FILELOCK_AVAILABLE = False
|
|
113
|
+
|
|
114
|
+
try: # POSIX fallback
|
|
115
|
+
import fcntl as _fcntl
|
|
116
|
+
|
|
117
|
+
_FCNTL_AVAILABLE = True
|
|
118
|
+
except ImportError: # pragma: no cover - Windows without pywin32
|
|
119
|
+
_fcntl = None # type: ignore[assignment]
|
|
120
|
+
_FCNTL_AVAILABLE = False
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class ArtifactStoreError(RuntimeError):
|
|
124
|
+
"""Base error for artifact-store failures (never silent data corruption)."""
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class ArtifactLockTimeoutError(ArtifactStoreError):
|
|
128
|
+
"""Another process/thread holds the task directory lock for too long."""
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class ArtifactOwnershipError(ArtifactStoreError):
|
|
132
|
+
"""A different work already owns this output directory — refusing to mix."""
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class UnresolvedRemoteSubmissionError(ArtifactStoreError):
|
|
136
|
+
"""A remote job may still be running; never bypass it by changing state."""
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class _ReentrantPathLock:
|
|
140
|
+
"""Process-wide reentrant lock for one lock path, with an OS-level guard.
|
|
141
|
+
|
|
142
|
+
Multiple :class:`ArtifactStore` instances (and multiple threads) for the
|
|
143
|
+
same resolved path share one object, so nested use inside one process is
|
|
144
|
+
reentrant instead of self-deadlocking, while a second *process* still waits
|
|
145
|
+
on the OS lock.
|
|
146
|
+
"""
|
|
147
|
+
|
|
148
|
+
_registry: dict[str, _ReentrantPathLock] = {}
|
|
149
|
+
_registry_guard = threading.Lock()
|
|
150
|
+
|
|
151
|
+
def __init__(self, path: Path) -> None:
|
|
152
|
+
self.path = path
|
|
153
|
+
self._mutex = threading.RLock()
|
|
154
|
+
self._depth = 0
|
|
155
|
+
self._file_handle: Any = None
|
|
156
|
+
self._filelock_handle: Any = None
|
|
157
|
+
self._backend: str = "none"
|
|
158
|
+
|
|
159
|
+
@classmethod
|
|
160
|
+
def for_path(cls, path: Path) -> _ReentrantPathLock:
|
|
161
|
+
key = str(path.expanduser().resolve())
|
|
162
|
+
with cls._registry_guard:
|
|
163
|
+
existing = cls._registry.get(key)
|
|
164
|
+
if existing is not None:
|
|
165
|
+
return existing
|
|
166
|
+
expanded = path.expanduser()
|
|
167
|
+
created = cls(expanded)
|
|
168
|
+
cls._registry[str(expanded.resolve())] = created
|
|
169
|
+
return created
|
|
170
|
+
|
|
171
|
+
# ------------------------------------------------------------ OS backend
|
|
172
|
+
def _acquire_os_lock(self, timeout: float | None) -> None:
|
|
173
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
174
|
+
deadline = None if timeout is None else time.monotonic() + max(0.0, timeout)
|
|
175
|
+
wait = 0.02
|
|
176
|
+
while True:
|
|
177
|
+
if _FILELOCK_AVAILABLE:
|
|
178
|
+
try:
|
|
179
|
+
handle = _filelock.FileLock(str(self.path), timeout=0)
|
|
180
|
+
except TypeError: # very old filelock
|
|
181
|
+
handle = _filelock.FileLock(str(self.path))
|
|
182
|
+
try:
|
|
183
|
+
handle.acquire(timeout=0)
|
|
184
|
+
except _filelock.Timeout:
|
|
185
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
186
|
+
raise ArtifactLockTimeoutError(
|
|
187
|
+
f"timed out waiting for artifact lock {self.path} "
|
|
188
|
+
f"({timeout}s); another YTK task is writing this directory"
|
|
189
|
+
) from None
|
|
190
|
+
time.sleep(wait)
|
|
191
|
+
continue
|
|
192
|
+
self._filelock_handle = handle
|
|
193
|
+
self._backend = "filelock"
|
|
194
|
+
return
|
|
195
|
+
if _FCNTL_AVAILABLE:
|
|
196
|
+
file_handle = open(self.path, "a+", encoding="utf-8") # noqa: SIM115 - held
|
|
197
|
+
try:
|
|
198
|
+
_fcntl.flock(file_handle.fileno(), _fcntl.LOCK_EX | _fcntl.LOCK_NB)
|
|
199
|
+
except OSError:
|
|
200
|
+
file_handle.close()
|
|
201
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
202
|
+
raise ArtifactLockTimeoutError(
|
|
203
|
+
f"timed out waiting for artifact lock {self.path} "
|
|
204
|
+
f"({timeout}s); another YTK task is writing this directory"
|
|
205
|
+
) from None
|
|
206
|
+
time.sleep(wait)
|
|
207
|
+
continue
|
|
208
|
+
self._file_handle = file_handle
|
|
209
|
+
self._backend = "fcntl"
|
|
210
|
+
return
|
|
211
|
+
# Last resort (no filelock, no fcntl): process-local only.
|
|
212
|
+
self._backend = "process"
|
|
213
|
+
LOGGER.warning(
|
|
214
|
+
"no filelock/fcntl available: artifact lock %s is process-local only",
|
|
215
|
+
self.path,
|
|
216
|
+
)
|
|
217
|
+
return
|
|
218
|
+
|
|
219
|
+
def _release_os_lock(self) -> None:
|
|
220
|
+
if self._filelock_handle is not None:
|
|
221
|
+
try:
|
|
222
|
+
self._filelock_handle.release()
|
|
223
|
+
except Exception: # noqa: BLE001 - release must never raise
|
|
224
|
+
LOGGER.warning("failed to release filelock %s", self.path)
|
|
225
|
+
self._filelock_handle = None
|
|
226
|
+
if self._file_handle is not None:
|
|
227
|
+
try:
|
|
228
|
+
_fcntl.flock(self._file_handle.fileno(), _fcntl.LOCK_UN)
|
|
229
|
+
except OSError:
|
|
230
|
+
pass
|
|
231
|
+
try:
|
|
232
|
+
self._file_handle.close()
|
|
233
|
+
except OSError:
|
|
234
|
+
pass
|
|
235
|
+
self._file_handle = None
|
|
236
|
+
self._backend = "none"
|
|
237
|
+
|
|
238
|
+
# ------------------------------------------------------------ public API
|
|
239
|
+
@contextmanager
|
|
240
|
+
def acquire(self, timeout: float | None) -> Iterator[None]:
|
|
241
|
+
acquired = False
|
|
242
|
+
if timeout is None or timeout < 0:
|
|
243
|
+
self._mutex.acquire()
|
|
244
|
+
acquired = True
|
|
245
|
+
else:
|
|
246
|
+
acquired = self._mutex.acquire(timeout=max(0.0, float(timeout)))
|
|
247
|
+
if not acquired:
|
|
248
|
+
raise ArtifactLockTimeoutError(
|
|
249
|
+
f"timed out waiting for artifact lock {self.path} ({timeout}s)"
|
|
250
|
+
)
|
|
251
|
+
try:
|
|
252
|
+
if self._depth == 0:
|
|
253
|
+
self._acquire_os_lock(timeout)
|
|
254
|
+
self._depth += 1
|
|
255
|
+
try:
|
|
256
|
+
yield
|
|
257
|
+
finally:
|
|
258
|
+
self._depth -= 1
|
|
259
|
+
if self._depth == 0:
|
|
260
|
+
self._release_os_lock()
|
|
261
|
+
finally:
|
|
262
|
+
self._mutex.release()
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
# --------------------------------------------------------------------------- #
|
|
266
|
+
# Small helpers
|
|
267
|
+
# --------------------------------------------------------------------------- #
|
|
268
|
+
def utc_now() -> str:
|
|
269
|
+
"""ISO-8601 UTC timestamp with second precision (manifest ``updated_at``)."""
|
|
270
|
+
return datetime.now(UTC).isoformat(timespec="seconds")
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def sha256_bytes(data: bytes) -> str:
|
|
274
|
+
return hashlib.sha256(data).hexdigest()
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def sha256_file(path: Path) -> str:
|
|
278
|
+
digest = hashlib.sha256()
|
|
279
|
+
with Path(path).open("rb") as handle:
|
|
280
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
281
|
+
digest.update(chunk)
|
|
282
|
+
return digest.hexdigest()
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def directory_file_count(path: Path) -> int:
|
|
286
|
+
"""Deterministic file count for a directory artifact (skill packages)."""
|
|
287
|
+
try:
|
|
288
|
+
return sum(1 for item in Path(path).rglob("*") if item.is_file())
|
|
289
|
+
except OSError:
|
|
290
|
+
return -1
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def directory_digest(path: Path) -> str | None:
|
|
294
|
+
"""Hash package-relative filenames and bytes; never follow symlinks."""
|
|
295
|
+
root = Path(path)
|
|
296
|
+
if root.is_symlink() or not root.is_dir():
|
|
297
|
+
return None
|
|
298
|
+
digest = hashlib.sha256()
|
|
299
|
+
files = 0
|
|
300
|
+
try:
|
|
301
|
+
for item in sorted(root.rglob("*")):
|
|
302
|
+
if item.is_symlink():
|
|
303
|
+
return None
|
|
304
|
+
if not item.is_file():
|
|
305
|
+
continue
|
|
306
|
+
digest.update(item.relative_to(root).as_posix().encode("utf-8"))
|
|
307
|
+
digest.update(b"\0")
|
|
308
|
+
digest.update(bytes.fromhex(sha256_file(item)))
|
|
309
|
+
files += 1
|
|
310
|
+
except (OSError, ValueError):
|
|
311
|
+
return None
|
|
312
|
+
return digest.hexdigest() if files else None
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _entry_extra(entry: OutputEntry) -> Mapping[str, Any]:
|
|
316
|
+
return entry.model_extra or {}
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _entry_is_directory_artifact(entry: OutputEntry, target: Path | None) -> bool:
|
|
320
|
+
kind = _entry_extra(entry).get("kind")
|
|
321
|
+
if kind == "directory":
|
|
322
|
+
return True
|
|
323
|
+
return target is not None and target.is_dir()
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def atomic_write_bytes(path: Path, data: bytes) -> None:
|
|
327
|
+
"""Write *data* to *path* atomically (same-directory temp + ``os.replace``)."""
|
|
328
|
+
path = Path(path)
|
|
329
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
330
|
+
fd, tmp_name = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".tmp")
|
|
331
|
+
try:
|
|
332
|
+
with os.fdopen(fd, "wb") as handle:
|
|
333
|
+
handle.write(data)
|
|
334
|
+
handle.flush()
|
|
335
|
+
try:
|
|
336
|
+
os.fsync(handle.fileno())
|
|
337
|
+
except OSError: # e.g. some network/CI filesystems
|
|
338
|
+
pass
|
|
339
|
+
os.replace(tmp_name, path)
|
|
340
|
+
except BaseException:
|
|
341
|
+
try:
|
|
342
|
+
os.unlink(tmp_name)
|
|
343
|
+
except OSError:
|
|
344
|
+
pass
|
|
345
|
+
raise
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _jsonable(payload: object) -> object:
|
|
349
|
+
if isinstance(payload, BaseModel):
|
|
350
|
+
return payload.model_dump(mode="json")
|
|
351
|
+
return payload
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _dump_json(payload: object) -> bytes:
|
|
355
|
+
data = json.dumps(_jsonable(payload), indent=2, ensure_ascii=False, sort_keys=True)
|
|
356
|
+
return (data + "\n").encode("utf-8")
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
# --------------------------------------------------------------------------- #
|
|
360
|
+
# Models
|
|
361
|
+
# --------------------------------------------------------------------------- #
|
|
362
|
+
class OutputEntry(BaseModel):
|
|
363
|
+
"""Per-format output record (contract §1.4 / plan §4.5)."""
|
|
364
|
+
|
|
365
|
+
model_config = ConfigDict(extra="allow")
|
|
366
|
+
|
|
367
|
+
status: OutputStatus = "pending"
|
|
368
|
+
path: str | None = None
|
|
369
|
+
sha256: str | None = None
|
|
370
|
+
validator_version: str | None = None
|
|
371
|
+
# ``not_applicable`` records its justification so resume can decide whether
|
|
372
|
+
# the judgment is still valid (reason + evidence fingerprint + checker ver.).
|
|
373
|
+
reason: str | None = None
|
|
374
|
+
reason_code: str | None = None
|
|
375
|
+
evidence_fingerprint: str | None = None
|
|
376
|
+
checker_version: str | None = None
|
|
377
|
+
updated_at: str | None = None
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
class ManifestV2(BaseModel):
|
|
381
|
+
"""The v2 task manifest; written last, never trusts an incomplete run."""
|
|
382
|
+
|
|
383
|
+
model_config = ConfigDict(extra="allow") # pipeline diagnostics ride along
|
|
384
|
+
|
|
385
|
+
schema_version: Literal[2] = 2
|
|
386
|
+
source_ids: list[str] = Field(default_factory=list)
|
|
387
|
+
source_revision: str | None = None
|
|
388
|
+
request_fingerprint: str = ""
|
|
389
|
+
stages: dict[str, StageStatus] = Field(default_factory=dict)
|
|
390
|
+
outputs: dict[str, OutputEntry] = Field(default_factory=dict)
|
|
391
|
+
missing: list[str] = Field(default_factory=list)
|
|
392
|
+
remote_tasks: dict[str, dict[str, Any]] = Field(default_factory=dict)
|
|
393
|
+
updated_at: str = ""
|
|
394
|
+
refresh_epoch: int = 0
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
# --------------------------------------------------------------------------- #
|
|
398
|
+
# Pure manifest helpers
|
|
399
|
+
# --------------------------------------------------------------------------- #
|
|
400
|
+
def new_manifest(
|
|
401
|
+
*,
|
|
402
|
+
source_ids: Sequence[str],
|
|
403
|
+
request_fingerprint: str,
|
|
404
|
+
source_revision: str | None = None,
|
|
405
|
+
refresh_epoch: int = 0,
|
|
406
|
+
) -> ManifestV2:
|
|
407
|
+
return ManifestV2(
|
|
408
|
+
source_ids=list(source_ids),
|
|
409
|
+
source_revision=source_revision,
|
|
410
|
+
request_fingerprint=request_fingerprint,
|
|
411
|
+
updated_at=utc_now(),
|
|
412
|
+
refresh_epoch=refresh_epoch,
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def mark_stage(manifest: ManifestV2, stage: str, status: StageStatus) -> ManifestV2:
|
|
417
|
+
updated = manifest.model_copy(deep=True)
|
|
418
|
+
updated.stages[stage] = status
|
|
419
|
+
return updated
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def record_output(
|
|
423
|
+
manifest: ManifestV2,
|
|
424
|
+
fmt: str,
|
|
425
|
+
*,
|
|
426
|
+
status: OutputStatus,
|
|
427
|
+
path: str | None = None,
|
|
428
|
+
sha256: str | None = None,
|
|
429
|
+
validator_version: str | None = None,
|
|
430
|
+
reason: str | None = None,
|
|
431
|
+
reason_code: str | None = None,
|
|
432
|
+
evidence_fingerprint: str | None = None,
|
|
433
|
+
checker_version: str | None = None,
|
|
434
|
+
) -> ManifestV2:
|
|
435
|
+
updated = manifest.model_copy(deep=True)
|
|
436
|
+
updated.outputs[fmt] = OutputEntry(
|
|
437
|
+
status=status,
|
|
438
|
+
path=path,
|
|
439
|
+
sha256=sha256,
|
|
440
|
+
validator_version=validator_version,
|
|
441
|
+
reason=reason,
|
|
442
|
+
reason_code=reason_code,
|
|
443
|
+
evidence_fingerprint=evidence_fingerprint,
|
|
444
|
+
checker_version=checker_version,
|
|
445
|
+
updated_at=utc_now(),
|
|
446
|
+
)
|
|
447
|
+
return updated
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def record_not_applicable(
|
|
451
|
+
manifest: ManifestV2,
|
|
452
|
+
fmt: str,
|
|
453
|
+
*,
|
|
454
|
+
reason_code: str,
|
|
455
|
+
evidence_fingerprint: str,
|
|
456
|
+
checker_version: str,
|
|
457
|
+
reason: str | None = None,
|
|
458
|
+
) -> ManifestV2:
|
|
459
|
+
"""Record a format-level NOT_APPLICABLE with its justification."""
|
|
460
|
+
return record_output(
|
|
461
|
+
manifest,
|
|
462
|
+
fmt,
|
|
463
|
+
status="not_applicable",
|
|
464
|
+
reason=reason,
|
|
465
|
+
reason_code=reason_code,
|
|
466
|
+
evidence_fingerprint=evidence_fingerprint,
|
|
467
|
+
checker_version=checker_version,
|
|
468
|
+
)
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def unresolved_remote_tasks(
|
|
472
|
+
manifest: ManifestV2, provider: str | None = None
|
|
473
|
+
) -> dict[str, dict[str, Any]]:
|
|
474
|
+
"""Remote tasks that may still be running (DISPATCHING / unknown id)."""
|
|
475
|
+
unresolved: dict[str, dict[str, Any]] = {}
|
|
476
|
+
for name, task in manifest.remote_tasks.items():
|
|
477
|
+
if provider is not None and name != provider:
|
|
478
|
+
continue
|
|
479
|
+
if not isinstance(task, Mapping):
|
|
480
|
+
unresolved[name] = {"status": "unknown"}
|
|
481
|
+
continue
|
|
482
|
+
status = str(task.get("status") or "unknown").strip().lower()
|
|
483
|
+
task_id = task.get("task_id")
|
|
484
|
+
if status in UNRESOLVED_REMOTE_STATUSES or (task_id in (None, "") and status not in RESOLVED_REMOTE_STATUSES):
|
|
485
|
+
unresolved[name] = dict(task)
|
|
486
|
+
return unresolved
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _upper_reason(entry: OutputEntry) -> str:
|
|
490
|
+
return str(entry.reason_code or entry.reason or "").strip().upper()
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def output_entry_claims_no_knowledge(entry: OutputEntry) -> bool:
|
|
494
|
+
"""True when a ``not_applicable`` entry records a NO_KNOWLEDGE verdict.
|
|
495
|
+
|
|
496
|
+
The v2 pipeline records ``reason_code="NO_KNOWLEDGE"`` for the content-level
|
|
497
|
+
no-knowledge verdict (R-11); older manifests may carry the legacy
|
|
498
|
+
``reason="NO_KNOWLEDGE: …"`` text instead.
|
|
499
|
+
"""
|
|
500
|
+
return _upper_reason(entry).startswith(REASON_NO_KNOWLEDGE)
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def manifest_recorded_outcome(manifest: ManifestV2) -> str:
|
|
504
|
+
"""The pipeline-authoritative outcome stored in the manifest (extra key)."""
|
|
505
|
+
extra = manifest.model_extra or {}
|
|
506
|
+
return str(extra.get("outcome") or "").strip().upper()
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _no_knowledge_evidence_valid(
|
|
510
|
+
manifest: ManifestV2, entries: Sequence[OutputEntry | None]
|
|
511
|
+
) -> bool:
|
|
512
|
+
"""Evidence/policy check for reusing a NO_KNOWLEDGE verdict (fail closed)."""
|
|
513
|
+
for entry in entries:
|
|
514
|
+
if entry is None:
|
|
515
|
+
return False
|
|
516
|
+
if entry.evidence_fingerprint and manifest.source_revision:
|
|
517
|
+
# When both sides recorded evidence it must agree. A manifest that
|
|
518
|
+
# predates the evidence field cannot disprove the verdict; the
|
|
519
|
+
# reason/validator checks below still apply.
|
|
520
|
+
if entry.evidence_fingerprint != manifest.source_revision:
|
|
521
|
+
return False
|
|
522
|
+
if entry.validator_version and entry.validator_version != OUTPUT_VALIDATOR_VERSION:
|
|
523
|
+
return False
|
|
524
|
+
return True
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def derive_overall_status(
|
|
528
|
+
manifest: ManifestV2, *, requested: Sequence[str] | None = None
|
|
529
|
+
) -> tuple[str, list[str]]:
|
|
530
|
+
"""Map a manifest to ``(overall_status, reasons)`` per plan §4.5.
|
|
531
|
+
|
|
532
|
+
``unknown`` (stage or unresolved remote submission) maps to ``blocked`` with
|
|
533
|
+
a reason code — it is never reported as complete.
|
|
534
|
+
"""
|
|
535
|
+
requested_formats = list(dict.fromkeys(requested or list(manifest.outputs)))
|
|
536
|
+
entries = {fmt: manifest.outputs.get(fmt) for fmt in requested_formats}
|
|
537
|
+
reasons: list[str] = []
|
|
538
|
+
|
|
539
|
+
unresolved = unresolved_remote_tasks(manifest)
|
|
540
|
+
for provider, task in unresolved.items():
|
|
541
|
+
status = str(task.get("status") or "unknown")
|
|
542
|
+
reasons.append(f"{REASON_REMOTE_SUBMISSION_UNKNOWN}:{provider}:{status}")
|
|
543
|
+
unknown_stages = sorted(stage for stage, status in manifest.stages.items() if status == "unknown")
|
|
544
|
+
for stage in unknown_stages:
|
|
545
|
+
reasons.append(f"{REASON_UNKNOWN_STAGE}:{stage}")
|
|
546
|
+
if unresolved or unknown_stages:
|
|
547
|
+
return OVERALL_BLOCKED, reasons
|
|
548
|
+
|
|
549
|
+
# NO_KNOWLEDGE is a content verdict and takes precedence over a format-level
|
|
550
|
+
# not-applicable judgment (plan §4.5 last paragraph, R-05/R-11). It is
|
|
551
|
+
# honoured from the pipeline-recorded ``outcome`` extra key OR the per-format
|
|
552
|
+
# ``reason_code``/``reason`` — but only while the recorded evidence and the
|
|
553
|
+
# judgment-policy version still match, otherwise the source is re-judged.
|
|
554
|
+
if requested_formats and all(
|
|
555
|
+
entry is not None and entry.status == "not_applicable" for entry in entries.values()
|
|
556
|
+
):
|
|
557
|
+
not_applicable_entries = [entries[fmt] for fmt in requested_formats]
|
|
558
|
+
claims_no_knowledge = manifest_recorded_outcome(manifest) == OVERALL_NO_KNOWLEDGE or all(
|
|
559
|
+
entry is not None and output_entry_claims_no_knowledge(entry)
|
|
560
|
+
for entry in not_applicable_entries
|
|
561
|
+
)
|
|
562
|
+
if claims_no_knowledge and _no_knowledge_evidence_valid(
|
|
563
|
+
manifest, not_applicable_entries
|
|
564
|
+
):
|
|
565
|
+
return OVERALL_NO_KNOWLEDGE, [
|
|
566
|
+
f"{REASON_NO_KNOWLEDGE}: no knowledge atoms for complete content"
|
|
567
|
+
]
|
|
568
|
+
reasons.append(
|
|
569
|
+
f"{REASON_OUTPUTS_NOT_APPLICABLE}: every requested format is not applicable"
|
|
570
|
+
)
|
|
571
|
+
return OVERALL_FAILED, reasons
|
|
572
|
+
|
|
573
|
+
failed = [
|
|
574
|
+
fmt
|
|
575
|
+
for fmt, entry in entries.items()
|
|
576
|
+
if entry is not None and entry.status == "failed"
|
|
577
|
+
]
|
|
578
|
+
blocked = [
|
|
579
|
+
fmt
|
|
580
|
+
for fmt, entry in entries.items()
|
|
581
|
+
if entry is not None and entry.status == "blocked"
|
|
582
|
+
]
|
|
583
|
+
failed_stages = sorted(stage for stage, status in manifest.stages.items() if status == "failed")
|
|
584
|
+
blocked_stages = sorted(stage for stage, status in manifest.stages.items() if status == "blocked")
|
|
585
|
+
if failed or failed_stages:
|
|
586
|
+
reasons.extend(f"{REASON_ARTIFACT_MISSING}:{fmt}" for fmt in failed)
|
|
587
|
+
reasons.extend(f"STAGE_FAILED:{stage}" for stage in failed_stages)
|
|
588
|
+
return OVERALL_FAILED, reasons
|
|
589
|
+
if blocked or blocked_stages:
|
|
590
|
+
reasons.extend(f"OUTPUT_BLOCKED:{fmt}" for fmt in blocked)
|
|
591
|
+
reasons.extend(f"STAGE_BLOCKED:{stage}" for stage in blocked_stages)
|
|
592
|
+
return OVERALL_BLOCKED, reasons
|
|
593
|
+
|
|
594
|
+
partial = (
|
|
595
|
+
[fmt for fmt, entry in entries.items() if entry is not None and entry.status == "partial"]
|
|
596
|
+
+ [fmt for fmt, entry in entries.items() if entry is not None and entry.status == "not_applicable"]
|
|
597
|
+
+ [fmt for fmt, entry in entries.items() if entry is not None and entry.status in {"pending", "skipped"}]
|
|
598
|
+
+ [fmt for fmt in requested_formats if entries.get(fmt) is None]
|
|
599
|
+
+ [stage for stage, status in manifest.stages.items() if status in {"pending", "running", "partial"}]
|
|
600
|
+
)
|
|
601
|
+
if partial:
|
|
602
|
+
reasons.extend(f"INCOMPLETE:{item}" for item in partial)
|
|
603
|
+
return OVERALL_PARTIAL, reasons
|
|
604
|
+
|
|
605
|
+
if manifest.missing:
|
|
606
|
+
reasons.extend(f"MISSING:{item}" for item in manifest.missing)
|
|
607
|
+
return OVERALL_PARTIAL, reasons
|
|
608
|
+
|
|
609
|
+
return OVERALL_COMPLETE, []
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
# --------------------------------------------------------------------------- #
|
|
613
|
+
# Artifact store
|
|
614
|
+
# --------------------------------------------------------------------------- #
|
|
615
|
+
class ArtifactStore:
|
|
616
|
+
"""Owns locked, atomic writes for one task output directory."""
|
|
617
|
+
|
|
618
|
+
def __init__(self, root: Path | str, *, lock_timeout: float = 10.0) -> None:
|
|
619
|
+
self.root = Path(root)
|
|
620
|
+
self.lock_timeout = float(lock_timeout)
|
|
621
|
+
self._lock = _ReentrantPathLock.for_path(self.root / f".ytk-artifacts{_LOCK_SUFFIX}")
|
|
622
|
+
|
|
623
|
+
# ------------------------------------------------------------- properties
|
|
624
|
+
@property
|
|
625
|
+
def lock_path(self) -> Path:
|
|
626
|
+
return self._lock.path
|
|
627
|
+
|
|
628
|
+
@property
|
|
629
|
+
def manifest_path(self) -> Path:
|
|
630
|
+
return self.root / MANIFEST_FILENAME
|
|
631
|
+
|
|
632
|
+
# ---------------------------------------------------------------- locking
|
|
633
|
+
@contextmanager
|
|
634
|
+
def locked(self, timeout: float | None = None) -> Iterator[None]:
|
|
635
|
+
with self._lock.acquire(self.lock_timeout if timeout is None else timeout):
|
|
636
|
+
yield
|
|
637
|
+
|
|
638
|
+
# ------------------------------------------------------------- path safety
|
|
639
|
+
def _safe_relpath(self, relpath: str | Path) -> Path:
|
|
640
|
+
candidate = Path(relpath)
|
|
641
|
+
if candidate.is_absolute() or candidate.drive:
|
|
642
|
+
raise ArtifactStoreError(f"artifact path must be relative: {relpath!r}")
|
|
643
|
+
if not candidate.parts or any(part == ".." for part in candidate.parts):
|
|
644
|
+
raise ArtifactStoreError(f"artifact path escapes the task directory: {relpath!r}")
|
|
645
|
+
return self.root / candidate
|
|
646
|
+
|
|
647
|
+
# ------------------------------------------------------------------ write
|
|
648
|
+
def write_bytes(
|
|
649
|
+
self, relpath: str | Path, data: bytes, *, sha256: str | None = None
|
|
650
|
+
) -> Path:
|
|
651
|
+
"""Validate then atomically write raw bytes; returns the written path."""
|
|
652
|
+
if not isinstance(data, (bytes, bytearray, memoryview)):
|
|
653
|
+
raise TypeError("artifact payload must be bytes")
|
|
654
|
+
payload = bytes(data)
|
|
655
|
+
digest = sha256_bytes(payload)
|
|
656
|
+
if sha256 is not None and sha256 != digest:
|
|
657
|
+
raise ArtifactStoreError(
|
|
658
|
+
f"refusing to write {relpath}: declared sha256 {sha256} != computed {digest}"
|
|
659
|
+
)
|
|
660
|
+
path = self._safe_relpath(relpath)
|
|
661
|
+
with self._lock.acquire(self.lock_timeout):
|
|
662
|
+
atomic_write_bytes(path, payload)
|
|
663
|
+
return path
|
|
664
|
+
|
|
665
|
+
def write_json(self, relpath: str | Path, payload: object) -> Path:
|
|
666
|
+
"""Serialise, validate the round-trip, then atomically replace the file."""
|
|
667
|
+
data = _dump_json(payload)
|
|
668
|
+
try:
|
|
669
|
+
json.loads(data)
|
|
670
|
+
except json.JSONDecodeError as exc: # pragma: no cover - defensive
|
|
671
|
+
raise ArtifactStoreError(f"refusing to write invalid JSON to {relpath}: {exc}") from exc
|
|
672
|
+
path = self._safe_relpath(relpath)
|
|
673
|
+
with self._lock.acquire(self.lock_timeout):
|
|
674
|
+
atomic_write_bytes(path, data)
|
|
675
|
+
return path
|
|
676
|
+
|
|
677
|
+
# ------------------------------------------------------------------- read
|
|
678
|
+
def read_json(self, relpath: str | Path) -> object | None:
|
|
679
|
+
path = self._safe_relpath(relpath)
|
|
680
|
+
try:
|
|
681
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
682
|
+
except FileNotFoundError:
|
|
683
|
+
return None
|
|
684
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
685
|
+
LOGGER.warning("artifact %s unreadable (%s): treating as missing", path, exc)
|
|
686
|
+
return None
|
|
687
|
+
|
|
688
|
+
def _read_manifest_unlocked(self) -> ManifestV2 | None:
|
|
689
|
+
try:
|
|
690
|
+
text = self.manifest_path.read_text(encoding="utf-8")
|
|
691
|
+
except FileNotFoundError:
|
|
692
|
+
return None
|
|
693
|
+
except OSError as exc:
|
|
694
|
+
LOGGER.warning("manifest %s unreadable (%s): treating as missing", self.manifest_path, exc)
|
|
695
|
+
return None
|
|
696
|
+
try:
|
|
697
|
+
payload = json.loads(text)
|
|
698
|
+
except json.JSONDecodeError as exc:
|
|
699
|
+
LOGGER.warning("manifest %s corrupt (%s): treating as missing", self.manifest_path, exc)
|
|
700
|
+
return None
|
|
701
|
+
if not isinstance(payload, dict):
|
|
702
|
+
return None
|
|
703
|
+
try:
|
|
704
|
+
manifest = ManifestV2.model_validate(payload)
|
|
705
|
+
except ValidationError as exc:
|
|
706
|
+
LOGGER.warning("manifest %s invalid (%s): treating as missing", self.manifest_path, exc)
|
|
707
|
+
return None
|
|
708
|
+
if manifest.schema_version != MANIFEST_SCHEMA_VERSION:
|
|
709
|
+
return None
|
|
710
|
+
return manifest
|
|
711
|
+
|
|
712
|
+
def read_manifest(self) -> ManifestV2 | None:
|
|
713
|
+
"""Return the manifest, or ``None`` when missing/corrupt (never raises)."""
|
|
714
|
+
return self._read_manifest_unlocked()
|
|
715
|
+
|
|
716
|
+
# ---------------------------------------------------------- manifest write
|
|
717
|
+
def write_manifest(self, manifest: ManifestV2) -> Path:
|
|
718
|
+
"""Write the manifest **last**, after validating every claimed output."""
|
|
719
|
+
if not isinstance(manifest, ManifestV2):
|
|
720
|
+
raise TypeError("write_manifest expects a ManifestV2")
|
|
721
|
+
with self._lock.acquire(self.lock_timeout):
|
|
722
|
+
existing = self._read_manifest_unlocked()
|
|
723
|
+
assert_manifest_ownership(existing, manifest)
|
|
724
|
+
prepared = self._prepare_manifest_for_write(manifest)
|
|
725
|
+
path = self.manifest_path
|
|
726
|
+
atomic_write_bytes(path, _dump_json(prepared))
|
|
727
|
+
return path
|
|
728
|
+
|
|
729
|
+
def _prepare_manifest_for_write(self, manifest: ManifestV2) -> ManifestV2:
|
|
730
|
+
prepared = manifest.model_copy(deep=True)
|
|
731
|
+
prepared.updated_at = prepared.updated_at or utc_now()
|
|
732
|
+
problems: list[str] = []
|
|
733
|
+
for fmt, entry in prepared.outputs.items():
|
|
734
|
+
if entry.status != "complete":
|
|
735
|
+
continue
|
|
736
|
+
target = self._output_path(entry.path)
|
|
737
|
+
if target is None or not target.exists():
|
|
738
|
+
problems.append(f"{fmt}: missing output file {entry.path!r}")
|
|
739
|
+
continue
|
|
740
|
+
if _entry_is_directory_artifact(entry, target):
|
|
741
|
+
if not target.is_dir():
|
|
742
|
+
problems.append(f"{fmt}: missing output directory {entry.path!r}")
|
|
743
|
+
continue
|
|
744
|
+
file_count = directory_file_count(target)
|
|
745
|
+
if file_count <= 0:
|
|
746
|
+
problems.append(f"{fmt}: empty output directory {entry.path!r}")
|
|
747
|
+
continue
|
|
748
|
+
recorded_count = _entry_extra(entry).get("file_count")
|
|
749
|
+
if recorded_count is not None:
|
|
750
|
+
try:
|
|
751
|
+
expected = int(recorded_count)
|
|
752
|
+
except (TypeError, ValueError):
|
|
753
|
+
problems.append(f"{fmt}: invalid file_count for {entry.path!r}")
|
|
754
|
+
continue
|
|
755
|
+
if expected != file_count:
|
|
756
|
+
problems.append(
|
|
757
|
+
f"{fmt}: file_count mismatch for {entry.path!r} "
|
|
758
|
+
f"({expected} != {file_count})"
|
|
759
|
+
)
|
|
760
|
+
continue
|
|
761
|
+
tree_digest = directory_digest(target)
|
|
762
|
+
recorded_digest = _entry_extra(entry).get("tree_sha256")
|
|
763
|
+
if tree_digest is None or (
|
|
764
|
+
recorded_digest is not None and recorded_digest != tree_digest
|
|
765
|
+
):
|
|
766
|
+
problems.append(f"{fmt}: directory digest mismatch for {entry.path!r}")
|
|
767
|
+
continue
|
|
768
|
+
entry.tree_sha256 = tree_digest # type: ignore[attr-defined]
|
|
769
|
+
entry.sha256 = None
|
|
770
|
+
if entry.path is None:
|
|
771
|
+
entry.path = (
|
|
772
|
+
str(target.relative_to(self.root))
|
|
773
|
+
if self._is_relative_to(target)
|
|
774
|
+
else str(target)
|
|
775
|
+
)
|
|
776
|
+
try:
|
|
777
|
+
entry.file_count = file_count # type: ignore[attr-defined]
|
|
778
|
+
except (AttributeError, ValueError): # pragma: no cover - pydantic extra
|
|
779
|
+
pass
|
|
780
|
+
continue
|
|
781
|
+
digest = sha256_file(target)
|
|
782
|
+
if entry.sha256 and entry.sha256 != digest:
|
|
783
|
+
problems.append(f"{fmt}: sha256 mismatch for {entry.path!r}")
|
|
784
|
+
continue
|
|
785
|
+
entry.sha256 = digest
|
|
786
|
+
if entry.path is None:
|
|
787
|
+
entry.path = str(target.relative_to(self.root)) if self._is_relative_to(target) else str(target)
|
|
788
|
+
if problems:
|
|
789
|
+
raise ArtifactStoreError(
|
|
790
|
+
"refusing to write a manifest that claims completion for unverifiable outputs: "
|
|
791
|
+
+ "; ".join(problems)
|
|
792
|
+
)
|
|
793
|
+
return prepared
|
|
794
|
+
|
|
795
|
+
def _output_path(self, recorded: str | None) -> Path | None:
|
|
796
|
+
if not recorded:
|
|
797
|
+
return None
|
|
798
|
+
candidate = Path(recorded)
|
|
799
|
+
target = candidate if candidate.is_absolute() else self.root / candidate
|
|
800
|
+
if not self._is_relative_to(target) or target.is_symlink():
|
|
801
|
+
return None
|
|
802
|
+
return target
|
|
803
|
+
|
|
804
|
+
def _is_relative_to(self, target: Path) -> bool:
|
|
805
|
+
try:
|
|
806
|
+
target.resolve().relative_to(self.root.resolve())
|
|
807
|
+
except (OSError, ValueError):
|
|
808
|
+
return False
|
|
809
|
+
return True
|
|
810
|
+
|
|
811
|
+
# --------------------------------------------------------------- validity
|
|
812
|
+
def output_valid(
|
|
813
|
+
self, fmt: str, *, validator_version: str, check_digest: bool = True
|
|
814
|
+
) -> bool:
|
|
815
|
+
"""True only when the manifest *and* the actual file both verify.
|
|
816
|
+
|
|
817
|
+
Missing manifest entry, non-complete status, validator mismatch, missing
|
|
818
|
+
file, or digest mismatch all return ``False`` (fail closed — a corrupt
|
|
819
|
+
artifact can never be used to skip work).
|
|
820
|
+
"""
|
|
821
|
+
manifest = self.read_manifest()
|
|
822
|
+
if manifest is None:
|
|
823
|
+
return False
|
|
824
|
+
entry = manifest.outputs.get(fmt)
|
|
825
|
+
if entry is None or entry.status != "complete":
|
|
826
|
+
return False
|
|
827
|
+
if not entry.validator_version or entry.validator_version != validator_version:
|
|
828
|
+
return False
|
|
829
|
+
target = self._output_path(entry.path)
|
|
830
|
+
if target is None or not target.exists():
|
|
831
|
+
return False
|
|
832
|
+
if _entry_is_directory_artifact(entry, target):
|
|
833
|
+
if not target.is_dir():
|
|
834
|
+
return False
|
|
835
|
+
file_count = directory_file_count(target)
|
|
836
|
+
if file_count <= 0:
|
|
837
|
+
return False
|
|
838
|
+
recorded_count = _entry_extra(entry).get("file_count")
|
|
839
|
+
if recorded_count is not None:
|
|
840
|
+
try:
|
|
841
|
+
if int(recorded_count) != file_count:
|
|
842
|
+
return False
|
|
843
|
+
except (TypeError, ValueError):
|
|
844
|
+
return False
|
|
845
|
+
recorded_digest = _entry_extra(entry).get("tree_sha256")
|
|
846
|
+
if not recorded_digest:
|
|
847
|
+
return False
|
|
848
|
+
if check_digest and directory_digest(target) != recorded_digest:
|
|
849
|
+
return False
|
|
850
|
+
return True
|
|
851
|
+
if not target.is_file():
|
|
852
|
+
return False
|
|
853
|
+
if not entry.sha256:
|
|
854
|
+
return False
|
|
855
|
+
if check_digest:
|
|
856
|
+
try:
|
|
857
|
+
if sha256_file(target) != entry.sha256:
|
|
858
|
+
return False
|
|
859
|
+
except OSError:
|
|
860
|
+
return False
|
|
861
|
+
return True
|
|
862
|
+
|
|
863
|
+
# ---------------------------------------------------------------- resume
|
|
864
|
+
def should_reuse_output(
|
|
865
|
+
self,
|
|
866
|
+
fmt: str,
|
|
867
|
+
*,
|
|
868
|
+
request_fingerprint: str,
|
|
869
|
+
validator_version: str,
|
|
870
|
+
source_revision: str | None = None,
|
|
871
|
+
refresh_epoch: int | None = None,
|
|
872
|
+
check_digest: bool = True,
|
|
873
|
+
) -> bool:
|
|
874
|
+
"""Decide whether an already-rendered format can be reused as-is."""
|
|
875
|
+
manifest = self.read_manifest()
|
|
876
|
+
if manifest is None:
|
|
877
|
+
return False
|
|
878
|
+
if not request_fingerprint or manifest.request_fingerprint != request_fingerprint:
|
|
879
|
+
return False
|
|
880
|
+
if source_revision is not None and manifest.source_revision not in (None, "") and (
|
|
881
|
+
manifest.source_revision != source_revision
|
|
882
|
+
):
|
|
883
|
+
return False
|
|
884
|
+
if refresh_epoch is not None and manifest.refresh_epoch != refresh_epoch:
|
|
885
|
+
return False
|
|
886
|
+
return self.output_valid(fmt, validator_version=validator_version, check_digest=check_digest)
|
|
887
|
+
|
|
888
|
+
def should_skip_format(
|
|
889
|
+
self,
|
|
890
|
+
fmt: str,
|
|
891
|
+
*,
|
|
892
|
+
request_fingerprint: str | None = None,
|
|
893
|
+
evidence_fingerprint: str | None = None,
|
|
894
|
+
checker_version: str | None = None,
|
|
895
|
+
validator_version: str | None = None,
|
|
896
|
+
source_revision: str | None = None,
|
|
897
|
+
) -> tuple[bool, str | None]:
|
|
898
|
+
"""Return ``(skip, reason_code)`` for one requested format.
|
|
899
|
+
|
|
900
|
+
* a verified complete output is skipped (reason ``None``);
|
|
901
|
+
* ``not_applicable`` is only skipped while its justification still holds
|
|
902
|
+
(same evidence fingerprint **and** checker version) — a changed
|
|
903
|
+
judgment policy or evidence forces a re-evaluation;
|
|
904
|
+
* blocked/unknown states are never skipped.
|
|
905
|
+
"""
|
|
906
|
+
manifest = self.read_manifest()
|
|
907
|
+
if manifest is None:
|
|
908
|
+
return (False, None)
|
|
909
|
+
entry = manifest.outputs.get(fmt)
|
|
910
|
+
if entry is None:
|
|
911
|
+
return (False, None)
|
|
912
|
+
if entry.status == "complete":
|
|
913
|
+
if validator_version is None:
|
|
914
|
+
return (False, None)
|
|
915
|
+
if request_fingerprint is not None and manifest.request_fingerprint != request_fingerprint:
|
|
916
|
+
return (False, None)
|
|
917
|
+
if self.should_reuse_output(
|
|
918
|
+
fmt,
|
|
919
|
+
request_fingerprint=manifest.request_fingerprint,
|
|
920
|
+
validator_version=validator_version,
|
|
921
|
+
source_revision=source_revision,
|
|
922
|
+
):
|
|
923
|
+
return (True, None)
|
|
924
|
+
return (False, None)
|
|
925
|
+
if entry.status == "not_applicable":
|
|
926
|
+
evidence_ok = bool(evidence_fingerprint) and (
|
|
927
|
+
entry.evidence_fingerprint == evidence_fingerprint
|
|
928
|
+
)
|
|
929
|
+
checker_ok = bool(checker_version) and (
|
|
930
|
+
entry.checker_version == checker_version
|
|
931
|
+
)
|
|
932
|
+
policy_ok = bool(validator_version) and (
|
|
933
|
+
entry.validator_version == validator_version
|
|
934
|
+
)
|
|
935
|
+
if output_entry_claims_no_knowledge(entry) and evidence_ok and (
|
|
936
|
+
checker_ok or policy_ok
|
|
937
|
+
):
|
|
938
|
+
# NO_KNOWLEDGE is a content verdict, not an "empty skill" failure:
|
|
939
|
+
# reuse it while the evidence and judgment policy version match.
|
|
940
|
+
return (True, entry.reason_code or REASON_NO_KNOWLEDGE)
|
|
941
|
+
if not evidence_ok or not checker_ok:
|
|
942
|
+
return (False, None) # cannot prove the judgment still holds
|
|
943
|
+
return (True, entry.reason_code or "NOT_APPLICABLE")
|
|
944
|
+
return (False, None)
|
|
945
|
+
|
|
946
|
+
# ----------------------------------------------------------- remote state
|
|
947
|
+
def unresolved_remote_tasks(self, provider: str | None = None) -> dict[str, dict[str, Any]]:
|
|
948
|
+
manifest = self.read_manifest()
|
|
949
|
+
if manifest is None:
|
|
950
|
+
return {}
|
|
951
|
+
return unresolved_remote_tasks(manifest, provider)
|
|
952
|
+
|
|
953
|
+
def has_unresolved_remote_submission(self, provider: str | None = None) -> bool:
|
|
954
|
+
return bool(self.unresolved_remote_tasks(provider))
|
|
955
|
+
|
|
956
|
+
def record_remote_task(
|
|
957
|
+
self,
|
|
958
|
+
provider: str,
|
|
959
|
+
*,
|
|
960
|
+
task_id: str | None = None,
|
|
961
|
+
status: str = "submitted",
|
|
962
|
+
deadline: float | None = None,
|
|
963
|
+
submitted_at: str | None = None,
|
|
964
|
+
**extra: Any,
|
|
965
|
+
) -> ManifestV2:
|
|
966
|
+
"""Insert/update one provider's remote task record (never deletes others)."""
|
|
967
|
+
with self._lock.acquire(self.lock_timeout):
|
|
968
|
+
manifest = self._read_manifest_unlocked() or new_manifest(
|
|
969
|
+
source_ids=[], request_fingerprint=""
|
|
970
|
+
)
|
|
971
|
+
updated = manifest.model_copy(deep=True)
|
|
972
|
+
record: dict[str, Any] = dict(updated.remote_tasks.get(provider) or {})
|
|
973
|
+
record.update(extra)
|
|
974
|
+
record["task_id"] = task_id
|
|
975
|
+
record["status"] = status
|
|
976
|
+
record["deadline"] = deadline
|
|
977
|
+
record["submitted_at"] = submitted_at or record.get("submitted_at")
|
|
978
|
+
updated.remote_tasks[provider] = record
|
|
979
|
+
updated.updated_at = utc_now()
|
|
980
|
+
atomic_write_bytes(self.manifest_path, _dump_json(updated))
|
|
981
|
+
return updated
|
|
982
|
+
|
|
983
|
+
# -------------------------------------------------------------- refresh
|
|
984
|
+
def invalidate_for_refresh(
|
|
985
|
+
self,
|
|
986
|
+
*,
|
|
987
|
+
refresh_epoch: int | None = None,
|
|
988
|
+
source_revision: str | None = None,
|
|
989
|
+
invalidate_stages: Sequence[str] = (),
|
|
990
|
+
invalidate_formats: Sequence[str] = (),
|
|
991
|
+
reason: str = REASON_REFRESH,
|
|
992
|
+
) -> ManifestV2:
|
|
993
|
+
"""Invalidate only what a refresh actually changed (plan §4.6/§9.1).
|
|
994
|
+
|
|
995
|
+
The request fingerprint is **not** rewritten here and remote-task records
|
|
996
|
+
are never deleted. When an unresolved remote submission exists, a
|
|
997
|
+
refresh that would invalidate acquisition-dependent stages is refused —
|
|
998
|
+
a new fingerprint must never be used to forget a possibly-running job.
|
|
999
|
+
"""
|
|
1000
|
+
with self._lock.acquire(self.lock_timeout):
|
|
1001
|
+
manifest = self._read_manifest_unlocked()
|
|
1002
|
+
if manifest is None:
|
|
1003
|
+
raise ArtifactStoreError(
|
|
1004
|
+
f"cannot refresh {self.root}: no valid manifest to invalidate"
|
|
1005
|
+
)
|
|
1006
|
+
unresolved = unresolved_remote_tasks(manifest)
|
|
1007
|
+
stages = set(invalidate_stages)
|
|
1008
|
+
if unresolved and (not stages or stages & _ACQUISITION_STAGES):
|
|
1009
|
+
providers = ", ".join(sorted(unresolved))
|
|
1010
|
+
raise UnresolvedRemoteSubmissionError(
|
|
1011
|
+
"refusing to invalidate acquisition state: remote submission(s) "
|
|
1012
|
+
f"still unresolved for {providers} — resolve or poll the existing "
|
|
1013
|
+
"task first, never re-submit it behind a new fingerprint"
|
|
1014
|
+
)
|
|
1015
|
+
updated = manifest.model_copy(deep=True)
|
|
1016
|
+
updated.refresh_epoch = (
|
|
1017
|
+
int(refresh_epoch) if refresh_epoch is not None else manifest.refresh_epoch + 1
|
|
1018
|
+
)
|
|
1019
|
+
if source_revision is not None:
|
|
1020
|
+
updated.source_revision = source_revision
|
|
1021
|
+
for stage in stages:
|
|
1022
|
+
updated.stages[stage] = "pending"
|
|
1023
|
+
for fmt in invalidate_formats:
|
|
1024
|
+
entry = updated.outputs.get(fmt) or OutputEntry()
|
|
1025
|
+
entry.status = "pending"
|
|
1026
|
+
entry.sha256 = None
|
|
1027
|
+
entry.validator_version = None
|
|
1028
|
+
entry.reason = reason
|
|
1029
|
+
entry.reason_code = reason
|
|
1030
|
+
entry.updated_at = utc_now()
|
|
1031
|
+
updated.outputs[fmt] = entry
|
|
1032
|
+
updated.updated_at = utc_now()
|
|
1033
|
+
# Remote-task records (including unknown ones) are copied verbatim.
|
|
1034
|
+
atomic_write_bytes(self.manifest_path, _dump_json(updated))
|
|
1035
|
+
return updated
|
|
1036
|
+
|
|
1037
|
+
# ------------------------------------------------------------- ownership
|
|
1038
|
+
def assert_owned_by(self, source_ids: Sequence[str]) -> None:
|
|
1039
|
+
"""Raise when an existing manifest belongs to a different work."""
|
|
1040
|
+
assert_manifest_ownership(self._read_manifest_unlocked(), source_ids)
|
|
1041
|
+
|
|
1042
|
+
|
|
1043
|
+
def assert_manifest_ownership(
|
|
1044
|
+
existing: ManifestV2 | None, incoming: ManifestV2 | Sequence[str]
|
|
1045
|
+
) -> None:
|
|
1046
|
+
"""Refuse to share one output directory between different works."""
|
|
1047
|
+
if existing is None or not existing.source_ids:
|
|
1048
|
+
return
|
|
1049
|
+
incoming_ids = (
|
|
1050
|
+
list(incoming.source_ids) if isinstance(incoming, ManifestV2) else list(incoming)
|
|
1051
|
+
)
|
|
1052
|
+
if not incoming_ids:
|
|
1053
|
+
return
|
|
1054
|
+
if not set(existing.source_ids) & set(incoming_ids):
|
|
1055
|
+
raise ArtifactOwnershipError(
|
|
1056
|
+
"output directory is owned by a different source: "
|
|
1057
|
+
f"existing={sorted(existing.source_ids)!r} incoming={sorted(incoming_ids)!r}; "
|
|
1058
|
+
"use a separate --out directory for each work"
|
|
1059
|
+
)
|
|
1060
|
+
|
|
1061
|
+
|
|
1062
|
+
__all__ = [
|
|
1063
|
+
"ArtifactLockTimeoutError",
|
|
1064
|
+
"ArtifactOwnershipError",
|
|
1065
|
+
"ArtifactStore",
|
|
1066
|
+
"ArtifactStoreError",
|
|
1067
|
+
"MANIFEST_FILENAME",
|
|
1068
|
+
"MANIFEST_SCHEMA_VERSION",
|
|
1069
|
+
"ManifestV2",
|
|
1070
|
+
"OVERALL_BLOCKED",
|
|
1071
|
+
"OVERALL_COMPLETE",
|
|
1072
|
+
"OVERALL_FAILED",
|
|
1073
|
+
"OVERALL_NO_KNOWLEDGE",
|
|
1074
|
+
"OVERALL_PARTIAL",
|
|
1075
|
+
"OUTPUT_VALIDATOR_VERSION",
|
|
1076
|
+
"OutputEntry",
|
|
1077
|
+
"OutputStatus",
|
|
1078
|
+
"REASON_ARTIFACT_CORRUPT",
|
|
1079
|
+
"REASON_ARTIFACT_MISSING",
|
|
1080
|
+
"REASON_DISPATCHING_UNKNOWN",
|
|
1081
|
+
"REASON_NO_KNOWLEDGE",
|
|
1082
|
+
"REASON_OUTPUTS_NOT_APPLICABLE",
|
|
1083
|
+
"REASON_REFRESH",
|
|
1084
|
+
"REASON_REMOTE_SUBMISSION_UNKNOWN",
|
|
1085
|
+
"REASON_UNKNOWN_STAGE",
|
|
1086
|
+
"REASON_VALIDATOR_MISMATCH",
|
|
1087
|
+
"RESOLVED_REMOTE_STATUSES",
|
|
1088
|
+
"StageName",
|
|
1089
|
+
"StageStatus",
|
|
1090
|
+
"UNRESOLVED_REMOTE_STATUSES",
|
|
1091
|
+
"UnresolvedRemoteSubmissionError",
|
|
1092
|
+
"assert_manifest_ownership",
|
|
1093
|
+
"atomic_write_bytes",
|
|
1094
|
+
"derive_overall_status",
|
|
1095
|
+
"directory_digest",
|
|
1096
|
+
"directory_file_count",
|
|
1097
|
+
"mark_stage",
|
|
1098
|
+
"manifest_recorded_outcome",
|
|
1099
|
+
"new_manifest",
|
|
1100
|
+
"output_entry_claims_no_knowledge",
|
|
1101
|
+
"record_not_applicable",
|
|
1102
|
+
"record_output",
|
|
1103
|
+
"sha256_bytes",
|
|
1104
|
+
"sha256_file",
|
|
1105
|
+
"unresolved_remote_tasks",
|
|
1106
|
+
"utc_now",
|
|
1107
|
+
]
|