kd-engine 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. kd_engine-1.0.0.dist-info/METADATA +19 -0
  2. kd_engine-1.0.0.dist-info/RECORD +118 -0
  3. kd_engine-1.0.0.dist-info/WHEEL +5 -0
  4. kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
  5. kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
  6. kd_engine-1.0.0.dist-info/top_level.txt +1 -0
  7. ytk/__init__.py +2 -0
  8. ytk/__main__.py +21 -0
  9. ytk/batch/__init__.py +39 -0
  10. ytk/batch/runner.py +1149 -0
  11. ytk/cache/__init__.py +34 -0
  12. ytk/cache/v2.py +562 -0
  13. ytk/cleaning/density.py +40 -0
  14. ytk/cleaning/fillers.py +18 -0
  15. ytk/cleaning/sentence_boundary.py +30 -0
  16. ytk/cleaning/service.py +62 -0
  17. ytk/cli.py +1281 -0
  18. ytk/commands/__init__.py +3 -0
  19. ytk/commands/merge.py +600 -0
  20. ytk/config.py +757 -0
  21. ytk/dedup/service.py +166 -0
  22. ytk/doctor.py +891 -0
  23. ytk/dotenv.py +334 -0
  24. ytk/errors.py +38 -0
  25. ytk/extraction/service.py +451 -0
  26. ytk/extraction/strategies/base.py +16 -0
  27. ytk/extraction/strategies/code_block.py +285 -0
  28. ytk/extraction/strategies/heuristic.py +791 -0
  29. ytk/extraction/strategies/local_model.py +56 -0
  30. ytk/extraction/text_context.py +177 -0
  31. ytk/extraction/visual_context.py +159 -0
  32. ytk/ingestion/adapters/__init__.py +1 -0
  33. ytk/ingestion/adapters/_common.py +865 -0
  34. ytk/ingestion/adapters/base.py +160 -0
  35. ytk/ingestion/adapters/bilibili.py +271 -0
  36. ytk/ingestion/adapters/douyin.py +445 -0
  37. ytk/ingestion/adapters/tiktok.py +102 -0
  38. ytk/ingestion/adapters/xiaohongshu.py +758 -0
  39. ytk/ingestion/adapters/youtube.py +74 -0
  40. ytk/ingestion/browser_acquired.py +220 -0
  41. ytk/ingestion/http_client.py +43 -0
  42. ytk/ingestion/local_import.py +202 -0
  43. ytk/ingestion/media.py +468 -0
  44. ytk/ingestion/providers/asr.py +902 -0
  45. ytk/ingestion/providers/base.py +13 -0
  46. ytk/ingestion/providers/dtk_client.py +816 -0
  47. ytk/ingestion/providers/minimax.py +350 -0
  48. ytk/ingestion/providers/transcript_api.py +336 -0
  49. ytk/ingestion/providers/xhs_client.py +646 -0
  50. ytk/ingestion/providers/ytdlp.py +400 -0
  51. ytk/ingestion/registry.py +189 -0
  52. ytk/ingestion/resolver.py +417 -0
  53. ytk/ingestion/search.py +56 -0
  54. ytk/ingestion/service.py +167 -0
  55. ytk/ingestion/subtitles.py +602 -0
  56. ytk/ingestion/transcript_cache.py +212 -0
  57. ytk/ingestion/url_parser.py +53 -0
  58. ytk/ingestion/video_frames.py +292 -0
  59. ytk/ingestion/ytdlp_acquire.py +754 -0
  60. ytk/logging.py +383 -0
  61. ytk/models/common.py +20 -0
  62. ytk/models/content.py +451 -0
  63. ytk/models/knowledge.py +43 -0
  64. ytk/models/migrations.py +262 -0
  65. ytk/models/skill.py +13 -0
  66. ytk/models/source.py +265 -0
  67. ytk/models/topic.py +18 -0
  68. ytk/models/transcript.py +30 -0
  69. ytk/ocr/__init__.py +65 -0
  70. ytk/ocr/engine.py +245 -0
  71. ytk/ocr/layout.py +347 -0
  72. ytk/ocr/pipeline.py +440 -0
  73. ytk/ocr/rapidocr_engine.py +144 -0
  74. ytk/pipeline/acquisition_checkpoint.py +240 -0
  75. ytk/pipeline/artifacts.py +18 -0
  76. ytk/pipeline/orchestrator.py +598 -0
  77. ytk/pipeline/processing_identity.py +18 -0
  78. ytk/pipeline/retry.py +728 -0
  79. ytk/pipeline/runners.py +578 -0
  80. ytk/pipeline/state.py +1107 -0
  81. ytk/pipeline/v2.py +1863 -0
  82. ytk/py.typed +1 -0
  83. ytk/quality/__init__.py +9 -0
  84. ytk/quality/completeness.py +260 -0
  85. ytk/renderers/base.py +9 -0
  86. ytk/renderers/citation_alignment.py +617 -0
  87. ytk/renderers/evidence_formatter.py +362 -0
  88. ytk/renderers/json_renderer.py +13 -0
  89. ytk/renderers/jsonl_renderer.py +18 -0
  90. ytk/renderers/knowledge_doc_renderer.py +1984 -0
  91. ytk/renderers/markdown_renderer.py +540 -0
  92. ytk/renderers/obsidian_renderer.py +138 -0
  93. ytk/renderers/skill_package.py +21 -0
  94. ytk/renderers/source_links.py +309 -0
  95. ytk/renderers/summary_renderer.py +163 -0
  96. ytk/renderers/v2.py +171 -0
  97. ytk/renderers/v2_jsonl.py +165 -0
  98. ytk/renderers/v2_markdown.py +477 -0
  99. ytk/renderers/v2_obsidian.py +113 -0
  100. ytk/renderers/v2_summary.py +220 -0
  101. ytk/segmentation/change_points.py +50 -0
  102. ytk/segmentation/embeddings.py +45 -0
  103. ytk/segmentation/service.py +318 -0
  104. ytk/skill_builder/__init__.py +1 -0
  105. ytk/skill_builder/builder.py +419 -0
  106. ytk/skill_builder/templates/SKILL.md.j2 +85 -0
  107. ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
  108. ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
  109. ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
  110. ytk/skill_builder/templates/references/commands.md.j2 +34 -0
  111. ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
  112. ytk/skill_builder/templates/references/configs.md.j2 +31 -0
  113. ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
  114. ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
  115. ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
  116. ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
  117. ytk/skill_builder/templates.py +29 -0
  118. ytk/text_normalization.py +44 -0
ytk/pipeline/v2.py ADDED
@@ -0,0 +1,1863 @@
1
+ """v2.0 multi-platform pipeline (T11.2 / T12.2 / T33.x integration).
2
+
3
+ Flow: resolve -> acquire (platform adapter) -> content bundle -> clean ->
4
+ segment -> extract -> render -> atomic manifest.
5
+
6
+ Two acquisition paths share everything after "content bundle":
7
+
8
+ * **live**: a :class:`~ytk.ingestion.adapters.base.PlatformAdapter` acquires the
9
+ source (yt-dlp / DTK / xiaohongshu-mcp / browser-derived fixture);
10
+ * **fixture-content**: a JSON descriptor (see :class:`FixtureContent`) describes
11
+ already-acquired content (real or recorded), including local media paths. This
12
+ is the offline/end-to-end path mandated by the plan (``--fixture-content``) and
13
+ the path used for acceptance evidence captured through the real browser.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import hashlib
19
+ import json
20
+ import logging
21
+ import os
22
+ import re as _re
23
+ import shutil
24
+ import tempfile
25
+ from collections.abc import Callable, Mapping, Sequence
26
+ from contextlib import nullcontext
27
+ from dataclasses import dataclass, field
28
+ from datetime import UTC
29
+ from pathlib import Path
30
+ from typing import TYPE_CHECKING, Any, Final, Literal, cast
31
+
32
+ from pydantic import BaseModel, ConfigDict, Field
33
+
34
+ from ytk.cleaning.service import CleaningService
35
+ from ytk.config import AppSettings
36
+ from ytk.errors import (
37
+ AnalysisBlockedError,
38
+ ModelUnavailableError,
39
+ ServiceNotConfiguredError,
40
+ UserFacingError,
41
+ )
42
+ from ytk.extraction.service import BlockExtraction, ExtractionService
43
+ from ytk.ingestion.registry import PlatformRegistry
44
+ from ytk.ingestion.resolver import (
45
+ access_tokens_from_url,
46
+ detect_platform,
47
+ extract_urls,
48
+ resolve_source,
49
+ )
50
+ from ytk.logging import redact_payload
51
+ from ytk.models.content import (
52
+ ContentBlock,
53
+ ContentBundle,
54
+ ContentCompleteness,
55
+ EvidenceRefV2,
56
+ ExtractionBundleV2,
57
+ KnowledgeAtomV2,
58
+ MediaAsset,
59
+ ProviderTrace,
60
+ dump_bundle_json,
61
+ make_block_id,
62
+ )
63
+ from ytk.models.source import AccessContext, ContentRef, ContentType, compute_source_revision
64
+ from ytk.pipeline.processing_identity import extraction_implementation_fingerprint
65
+ from ytk.segmentation.service import TopicSegmentationService
66
+
67
+ if TYPE_CHECKING: # pragma: no cover - typing only (avoids the registry import cycle)
68
+ from ytk.ingestion.adapters.base import PlatformAdapter
69
+
70
+
71
+ #: Metadata keys whose very name marks the value as credential-like. Value-shape
72
+ #: redaction (URLs, token patterns) alone is not enough: a bare ``SESSDATA`` or
73
+ #: ``signature`` string value survives ``redact_payload`` (adversarial review P0).
74
+ _SENSITIVE_META_KEY = _re.compile(
75
+ r"(xsec|mstoken|a_bogus|w_rid|bearer|api[-_]?key|token|cookie|session|sess|csrf|jct|"
76
+ r"secret|signature|authorization|password|credential)",
77
+ _re.IGNORECASE,
78
+ )
79
+ _REDACTED = "***REDACTED***"
80
+
81
+
82
+ def _sanitize_metadata(value: object) -> object:
83
+ """Redact credentials by key name AND by value shape, recursively."""
84
+ if isinstance(value, dict):
85
+ return {
86
+ key: _REDACTED if _SENSITIVE_META_KEY.search(str(key)) else _sanitize_metadata(item)
87
+ for key, item in value.items()
88
+ }
89
+ if isinstance(value, (list, tuple)):
90
+ return [_sanitize_metadata(item) for item in value]
91
+ if isinstance(value, str):
92
+ redacted = redact_payload(value)
93
+ return redacted if isinstance(redacted, str) else str(redacted)
94
+ return value
95
+
96
+ LOGGER = logging.getLogger(__name__)
97
+
98
+ OUTCOME_COMPLETE: Final = "COMPLETE"
99
+ OUTCOME_PARTIAL: Final = "PARTIAL"
100
+ OUTCOME_BLOCKED: Final = "BLOCKED"
101
+ OUTCOME_FAILED: Final = "FAILED"
102
+ OUTCOME_NO_KNOWLEDGE: Final = "NO_KNOWLEDGE"
103
+
104
+ Outcome = Literal["COMPLETE", "PARTIAL", "BLOCKED", "FAILED", "NO_KNOWLEDGE"]
105
+
106
+ #: formats that need real knowledge (as opposed to "not applicable" outcomes)
107
+ KNOWLEDGE_FORMATS = frozenset({"json", "md", "md_notes", "obsidian", "summary", "jsonl", "doc"})
108
+
109
+
110
+ class FixtureContent(BaseModel):
111
+ """``--fixture-content`` descriptor: already-acquired content, no network."""
112
+
113
+ model_config = ConfigDict(extra="forbid")
114
+
115
+ schema_version: Literal[2] = 2
116
+ source: ContentRef
117
+ metadata: dict[str, object] = Field(default_factory=dict)
118
+ blocks: list[ContentBlock] = Field(default_factory=list)
119
+ media: list[dict[str, object]] = Field(default_factory=list)
120
+ subtitle_text: str | None = None
121
+ completeness: ContentCompleteness = Field(
122
+ default_factory=lambda: ContentCompleteness(status="complete", checked=["fixture_content"])
123
+ )
124
+ provider_trace: list[ProviderTrace] = Field(default_factory=list)
125
+ chapters: list[dict[str, object]] = Field(default_factory=list)
126
+
127
+ @property
128
+ def content_type(self) -> ContentType:
129
+ return self.source.content_type
130
+
131
+
132
+ class _DeterministicEmbeddingClient:
133
+ model_name = "deterministic-local"
134
+
135
+ def __init__(self, dimensions: int = 12) -> None:
136
+ self._dimensions = dimensions
137
+
138
+ def encode(self, texts: Sequence[str]) -> list[list[float]]:
139
+ import re
140
+
141
+ vectors = []
142
+ for text in texts:
143
+ vector = [0.0] * self._dimensions
144
+ for word in re.findall(r"[a-z0-9]+", text.casefold()):
145
+ vector[sum(ord(char) for char in word) % self._dimensions] += 1.0
146
+ vectors.append(vector)
147
+ return vectors
148
+
149
+
150
+ @dataclass
151
+ class V2RunResult:
152
+ """What one v2 run produced (kept out of the JSON models for flexibility)."""
153
+
154
+ bundle: ExtractionBundleV2
155
+ output_dir: Path
156
+ formats: set[str]
157
+ outcome: Outcome
158
+ format_status: dict[str, str] = field(default_factory=dict)
159
+ reasons: list[str] = field(default_factory=list)
160
+ manifest_path: Path | None = None
161
+ failed_formats: dict[str, str] = field(default_factory=dict)
162
+
163
+
164
+ def _sha256_bytes(data: bytes) -> str:
165
+ return hashlib.sha256(data).hexdigest()
166
+
167
+
168
+ def _sha256_file(path: Path) -> str:
169
+ digest = hashlib.sha256()
170
+ with path.open("rb") as handle:
171
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
172
+ digest.update(chunk)
173
+ return digest.hexdigest()
174
+
175
+
176
+ def _share_url(value: str) -> str:
177
+ try:
178
+ return extract_urls(value)[0]
179
+ except ValueError:
180
+ return value
181
+
182
+
183
+ class MultiPlatformPipeline:
184
+ """Assemble and persist a v2 knowledge product for any registered platform."""
185
+
186
+ def __init__(
187
+ self,
188
+ *,
189
+ settings: AppSettings | None = None,
190
+ registry: PlatformRegistry | None = None,
191
+ access: AccessContext | None = None,
192
+ renderers: Mapping[str, Callable[..., None]] | None = None,
193
+ asr_runner: Callable[..., list[ContentBlock]] | None = None,
194
+ ocr_runner: Callable[..., list[ContentBlock]] | None = None,
195
+ artifact_store_factory: Callable[[Path], Any] | None = None,
196
+ embedding_client: Any | None = None,
197
+ coarse_atoms: bool = False,
198
+ ) -> None:
199
+ self.settings = settings or AppSettings()
200
+ self.registry = registry or PlatformRegistry.default()
201
+ self.access = access or AccessContext()
202
+ self.renderers = dict(renderers or {})
203
+ self.asr_runner = asr_runner
204
+ self.ocr_runner = ocr_runner
205
+ if artifact_store_factory is None:
206
+ from ytk.pipeline.runners import make_artifact_store_factory
207
+
208
+ artifact_store_factory = make_artifact_store_factory()
209
+ self.artifact_store_factory = artifact_store_factory
210
+ self.cleaning = CleaningService()
211
+ self.segmentation = TopicSegmentationService(
212
+ embedding_client=embedding_client or _DeterministicEmbeddingClient()
213
+ )
214
+ self.extraction = ExtractionService()
215
+ # 粗粒度分段:默认关闭。开启后,只有在完全抽不出原子时才生效。
216
+ self.coarse_atoms = bool(coarse_atoms)
217
+
218
+ # ------------------------------------------------------------------ run
219
+ def run(
220
+ self,
221
+ url: str,
222
+ *,
223
+ output_dir: Path,
224
+ formats: Sequence[str] | None = None,
225
+ fixture_content: Path | None = None,
226
+ refresh: bool = False,
227
+ continue_on_error: bool = True,
228
+ formats_explicit: bool = False,
229
+ doc_language: str | None = None,
230
+ platform: str | None = None,
231
+ content_type: ContentType | None = None,
232
+ preserve_existing_formats: bool = False,
233
+ ) -> V2RunResult:
234
+ output_dir = Path(output_dir)
235
+ output_dir.mkdir(parents=True, exist_ok=True)
236
+ store = self._store(output_dir)
237
+ lock = store.locked(store.lock_timeout) if store is not None else nullcontext()
238
+ with lock: # hold the directory lock for the WHOLE task (TOCTOU fix, A39)
239
+ previous_manifest = self._read_previous_manifest(output_dir)
240
+ return self._run_inner(
241
+ url,
242
+ output_dir=output_dir,
243
+ formats=formats,
244
+ fixture_content=fixture_content,
245
+ refresh=refresh,
246
+ continue_on_error=continue_on_error,
247
+ formats_explicit=formats_explicit,
248
+ doc_language=doc_language,
249
+ platform=platform,
250
+ content_type=content_type,
251
+ previous_manifest=previous_manifest,
252
+ preserve_existing_formats=preserve_existing_formats,
253
+ )
254
+
255
+ def _run_inner(
256
+ self,
257
+ url: str,
258
+ *,
259
+ output_dir: Path,
260
+ formats: Sequence[str] | None,
261
+ fixture_content: Path | None,
262
+ refresh: bool,
263
+ continue_on_error: bool,
264
+ formats_explicit: bool,
265
+ doc_language: str | None,
266
+ platform: str | None,
267
+ content_type: ContentType | None,
268
+ previous_manifest: Any | None,
269
+ preserve_existing_formats: bool = False,
270
+ ) -> V2RunResult:
271
+ resolved_tokens: dict[str, str] = {}
272
+
273
+ def capture_resolved_url(resolved_url: str) -> None:
274
+ resolved_tokens.update(access_tokens_from_url(resolved_url))
275
+
276
+ task_access = self._task_access(url, resolved_url_callback=capture_resolved_url)
277
+ deadline = self._make_deadline()
278
+
279
+ acquisition: Any | None = None
280
+ reused_content = False
281
+ if fixture_content is not None:
282
+ fixture = self._load_fixture_content(Path(fixture_content))
283
+ ref = fixture.source
284
+ self._assert_output_ownership(output_dir, previous_manifest, ref)
285
+ self._assert_no_unresolved_remote(previous_manifest)
286
+ if fixture.metadata.get("acquisition_mode") == "browser_video_live":
287
+ content, acquisition = self._browser_content_from_fixture(fixture, output_dir, url)
288
+ else:
289
+ content = self._content_from_fixture(fixture, output_dir)
290
+ else:
291
+ ref = self._resolve(url, platform=platform, content_type=content_type, access=task_access)
292
+ task_access = task_access.model_copy(deep=True, update={
293
+ "tokens": {**task_access.tokens, **resolved_tokens}, "resolved_url_callback": None,
294
+ })
295
+ self._assert_output_ownership(output_dir, previous_manifest, ref, allow_unresolved_variant=True)
296
+ self._assert_no_unresolved_remote(previous_manifest)
297
+ cached = self._reusable_content(output_dir, previous_manifest, ref) if preserve_existing_formats and not refresh else None
298
+ if cached is not None:
299
+ content = cached
300
+ ref = content.source
301
+ reused_content = True
302
+ else:
303
+ content, acquisition = self._acquired_content(
304
+ ref, output_dir, previous_manifest, task_access, refresh=refresh,
305
+ content_type=content_type,
306
+ )
307
+ ref = content.source
308
+
309
+ requested = self._resolve_formats(ref, formats)
310
+ if formats_explicit and not requested:
311
+ raise UserFacingError("explicitly requested format set is empty")
312
+
313
+ # ASR / OCR enrich the content *before* the revision is frozen so that
314
+ # evidence ids and the revision stay consistent.
315
+ if not reused_content:
316
+ self._check_deadline(deadline, "ocr")
317
+ content = self._maybe_ocr(content, output_dir)
318
+ self._check_deadline(deadline, "frame-ocr")
319
+ content = self._maybe_frame_ocr(content, output_dir)
320
+ self._check_deadline(deadline, "asr")
321
+ content = self._maybe_asr(content, output_dir)
322
+ if content.metadata.get("acquisition_mode") == "browser_video_live":
323
+ from ytk.ingestion.browser_acquired import assess_browser_transcript
324
+ content = assess_browser_transcript(content)
325
+ # Re-derive completeness from the evidence (never trust a declared
326
+ # "complete"): truncated subtitles, failed ASR chunks, cover-only and
327
+ # unverified narration must downgrade the run (T22.2 / A29).
328
+ if acquisition is not None:
329
+ content = self._assess_completeness(acquisition, content)
330
+ elif content.metadata.get("acquisition_mode") == "user_import":
331
+ from ytk.ingestion.local_import import assess_import
332
+ content = assess_import(content)
333
+ content = content.model_copy(update={"source_revision": compute_source_revision(
334
+ content.source.source_id,
335
+ {
336
+ "blocks": [block.model_dump(mode="json", exclude={"cleaned_text"}) for block in sorted(content.blocks, key=lambda block: (block.order, block.block_id))],
337
+ "assets": [(asset.media_type, asset.order, asset.sha256) for asset in sorted(content.assets, key=lambda asset: (asset.order, asset.asset_id))],
338
+ },
339
+ )})
340
+
341
+ bundle = self._sanitize_bundle(self._assemble(ref, content))
342
+ outcome, format_status, format_reasons, reasons = self._decide_outcomes(bundle, requested)
343
+
344
+ self._check_deadline(deadline, "render")
345
+ written = self._render(bundle, requested, output_dir, format_status, doc_language=doc_language)
346
+ if format_status.get("doc") == "blocked":
347
+ try:
348
+ status = json.loads((output_dir / ".knowledge-doc-status.json").read_text(encoding="utf-8"))
349
+ format_reasons["doc"] = str(status.get("reason") or "DOCUMENT_UNVERIFIED")
350
+ except (OSError, ValueError, TypeError):
351
+ format_reasons["doc"] = "DOCUMENT_UNVERIFIED"
352
+ reasons.append("doc BLOCKED: " + format_reasons["doc"])
353
+ outcome = cast(Outcome, self._adjust_outcome_after_render(outcome, format_status, requested))
354
+ manifest_formats = set(requested)
355
+ if (
356
+ preserve_existing_formats
357
+ and previous_manifest is not None
358
+ and previous_manifest.source_revision == content.source_revision
359
+ and bundle.knowledge
360
+ ):
361
+ store = self._store(output_dir)
362
+ if store is not None:
363
+ for fmt, entry in previous_manifest.outputs.items():
364
+ same_settings = (
365
+ fmt == "json" or getattr(previous_manifest, "render_fingerprints", {}).get(fmt)
366
+ == render_request_fingerprint(fmt, self.settings, doc_language, ref.platform)
367
+ )
368
+ if fmt not in requested and same_settings and store.output_valid(fmt, validator_version=FORMAT_VALIDATOR_VERSION):
369
+ manifest_formats.add(fmt)
370
+ format_status[fmt] = "complete"
371
+ written[fmt] = output_dir / entry.path
372
+ self._remove_stale_artifacts(previous_manifest, written, output_dir)
373
+ manifest_path = self._write_manifest(
374
+ bundle,
375
+ output_dir,
376
+ requested=manifest_formats,
377
+ written=written,
378
+ outcome=outcome,
379
+ format_status=format_status,
380
+ format_reasons=format_reasons,
381
+ reasons=reasons,
382
+ refresh=refresh,
383
+ previous_manifest=previous_manifest,
384
+ doc_language=doc_language,
385
+ acquisition_mode=("user-import" if content.metadata.get("acquisition_mode") == "user_import"
386
+ else ("browser-video-live" if content.metadata.get("acquisition_mode") == "browser_video_live"
387
+ else ("fixture-content" if fixture_content is not None else "platform"))),
388
+ )
389
+ return V2RunResult(
390
+ bundle=bundle,
391
+ output_dir=output_dir,
392
+ formats=requested,
393
+ outcome=outcome,
394
+ format_status=format_status,
395
+ reasons=reasons,
396
+ manifest_path=manifest_path,
397
+ )
398
+
399
+ # ------------------------------------------------------------ resolution
400
+ def _resolve(self, url: str, *, platform: str | None, content_type: ContentType | None,
401
+ access: AccessContext | None = None) -> ContentRef:
402
+ from ytk.ingestion.registry import _REGISTRATION_ORDER # local: avoid import cycle
403
+
404
+ detected: str | None = detect_platform(_share_url(url))
405
+ if detected is None:
406
+ try:
407
+ detected = resolve_source(url, content_type=content_type).platform
408
+ except UserFacingError:
409
+ detected = platform
410
+ platform = platform or detected
411
+ if platform is None:
412
+ raise UserFacingError(f"cannot determine the platform of {url!r}")
413
+ if platform not in _REGISTRATION_ORDER:
414
+ raise UserFacingError(f"unsupported platform {platform!r}")
415
+ adapter = self.registry.get(platform).adapter
416
+ if adapter is not None:
417
+ # PlatformRegistration.adapter is stored as ``object`` (pydantic
418
+ # cannot validate a Protocol); the registry only ever installs
419
+ # PlatformAdapter instances.
420
+ return cast("PlatformAdapter", adapter).resolve(url, access or self.access)
421
+ return resolve_source(url, platform=platform, content_type=content_type) # type: ignore[arg-type]
422
+
423
+ def _convert_sidecar_error(self, exc: Exception, output_dir: Path) -> Exception | None:
424
+ """Convert sidecar typed errors into blocked outcomes WITH state recorded.
425
+
426
+ A DTK task timeout carries a resumable task id: recording it here is what
427
+ makes the unresolved-remote guard (A36) real on the live path — the next
428
+ run sees the record and refuses to re-dispatch instead of starting over.
429
+ """
430
+ name = type(exc).__name__
431
+ if name not in {
432
+ "DtkTaskTimeoutError",
433
+ "DtkAuthRequiredError",
434
+ "DtkSubmissionUnknownError",
435
+ "DtkTaskFailedError",
436
+ "DtkServiceUnavailableError",
437
+ "DtkRateLimitedError",
438
+ "DtkContentUnavailableError",
439
+ "DtkApiChangedError",
440
+ "XhsUrlExpiredError",
441
+ }:
442
+ return None
443
+ if name == "DtkTaskTimeoutError":
444
+ store = self._store(output_dir)
445
+ if store is not None:
446
+ try:
447
+ store.record_remote_task(
448
+ "dtk",
449
+ task_id=getattr(exc, "task_id", None),
450
+ status="running",
451
+ deadline=getattr(exc, "deadline", None),
452
+ )
453
+ except Exception as record_exc: # pragma: no cover - never crash on state
454
+ LOGGER.warning("could not record remote task: %s", record_exc)
455
+ elif name in {"DtkAuthRequiredError", "DtkRateLimitedError", "DtkContentUnavailableError", "DtkTaskFailedError"}:
456
+ store = self._store(output_dir)
457
+ if store is not None:
458
+ saved = store.read_manifest()
459
+ task = saved.remote_tasks.get("dtk", {}) if saved is not None else {}
460
+ if not task.get("task_id") or name == "DtkTaskFailedError":
461
+ store.record_remote_task("dtk", status="failed", task_id=task.get("task_id"))
462
+ reason = getattr(exc, "reason_code", None) or name
463
+ return AnalysisBlockedError(f"{reason}: {exc}")
464
+
465
+ def _make_deadline(self) -> Any | None:
466
+ seconds = float(getattr(self.settings, "task_deadline_s", 0) or 0)
467
+ if seconds <= 0:
468
+ return None
469
+ from ytk.pipeline.retry import TaskDeadline
470
+
471
+ return TaskDeadline(seconds)
472
+
473
+ @staticmethod
474
+ def _check_deadline(deadline: Any | None, stage: str) -> None:
475
+ if deadline is None:
476
+ return
477
+ try:
478
+ deadline.check()
479
+ except Exception as exc:
480
+ from ytk.errors import AnalysisBlockedError
481
+
482
+ raise AnalysisBlockedError(f"TASK_DEADLINE_EXCEEDED at stage {stage}: {exc}") from exc
483
+
484
+ def _task_access(self, url: str, *, resolved_url_callback: Callable[[str], None] | None = None) -> AccessContext:
485
+ """Build the per-task credential context from settings + the share URL.
486
+
487
+ Platform isolation matters here (plan §4.6): the legacy global YouTube
488
+ cookie/proxy must never leak to another platform, and an XHS ``xsec_token``
489
+ carried by the share URL belongs in the protected access context, never in
490
+ a public artifact.
491
+ """
492
+ candidate = _share_url(url)
493
+ platform = detect_platform(candidate)
494
+ cookies_file = self.access.cookies_file
495
+ cookies_from_browser = self.access.cookies_from_browser
496
+ proxy = self.access.proxy
497
+ headers = dict(self.access.headers)
498
+ tokens = dict(self.access.tokens)
499
+ if platform is not None:
500
+ try:
501
+ cookies_file = self.settings.cookies_file_for(platform) or cookies_file
502
+ cookies_from_browser = (
503
+ self.settings.cookies_from_browser_for(platform) or cookies_from_browser
504
+ )
505
+ proxy = self.settings.proxy_for(platform) or proxy
506
+ except Exception: # pragma: no cover - defensive
507
+ pass
508
+ try:
509
+ tokens.update(access_tokens_from_url(candidate))
510
+ except Exception: # pragma: no cover - defensive
511
+ pass
512
+ return AccessContext(
513
+ cookies_file=cookies_file,
514
+ cookies_from_browser=cookies_from_browser,
515
+ proxy=proxy,
516
+ headers=headers,
517
+ tokens=tokens,
518
+ resolved_url_callback=resolved_url_callback,
519
+ )
520
+
521
+ def _acquire(
522
+ self, ref: ContentRef, work_dir: Path | None = None, access: AccessContext | None = None,
523
+ refresh: bool = False,
524
+ ) -> Any:
525
+ from ytk.ingestion.adapters.base import AcquisitionContext
526
+
527
+ adapter = self.registry.require_adapter(ref.platform)
528
+ if work_dir is None:
529
+ work_dir = Path(".")
530
+ work_dir.mkdir(parents=True, exist_ok=True)
531
+ task_options: dict[str, Any] = {}
532
+ if ref.platform == "douyin":
533
+ from ytk.pipeline.state import new_manifest
534
+
535
+ store = self._store(work_dir.parent.parent)
536
+ if store is None:
537
+ raise UserFacingError("cannot safely persist the remote submission state")
538
+ saved = store.read_manifest()
539
+ if saved is None:
540
+ saved = new_manifest(source_ids=[ref.source_id], request_fingerprint="")
541
+ store.write_manifest(saved)
542
+ remote = saved.remote_tasks.get("dtk", {})
543
+ if remote.get("status") not in {"failed", "cancelled", "canceled"} and not (
544
+ refresh and remote.get("status") == "complete"
545
+ ):
546
+ task_options["remote_task_id"] = remote.get("task_id")
547
+
548
+ def record_remote(status: str, task_id: str | None) -> None:
549
+ store.record_remote_task("dtk", status=status, task_id=task_id)
550
+
551
+ task_options["remote_task_callback"] = record_remote
552
+ context = AcquisitionContext(
553
+ access=access or self.access,
554
+ work_dir=str(work_dir),
555
+ languages=getattr(self.settings, "transcript_languages", ("en", "zh-Hans", "zh")),
556
+ asr_enabled=True,
557
+ ocr_enabled=bool(getattr(self.settings, "ocr_enabled", False)),
558
+ media_budget_mb=self.settings.max_media_mb,
559
+ frame_budget={"max_frames": self.settings.frame_budget},
560
+ **task_options,
561
+ )
562
+ return adapter.acquire(ref, context)
563
+
564
+ # --------------------------------------------------------------- content
565
+ def _processing_fingerprint(self) -> str:
566
+ from importlib.metadata import PackageNotFoundError, version
567
+
568
+ from ytk.pipeline.runners import _asr_model_fingerprint, resolve_asr_model
569
+ from ytk.ingestion.providers.asr import asr_decoding_profile
570
+
571
+ model: object = None
572
+ try:
573
+ selected = resolve_asr_model(self.settings)
574
+ model = {"path": selected, "files": _asr_model_fingerprint(selected)}
575
+ except (ModelUnavailableError, OSError):
576
+ pass
577
+ ocr: object = None
578
+ if self.settings.ocr_enabled:
579
+ from ytk.ocr.engine import resolve_model_paths
580
+
581
+ try:
582
+ ocr = resolve_model_paths(self.settings.ocr_model_dir).fingerprint()
583
+ except (ModelUnavailableError, OSError):
584
+ pass
585
+ payload = {
586
+ "asr_backend": self.settings.asr_backend,
587
+ "asr_model": self.settings.asr_model,
588
+ "asr_decoding": asr_decoding_profile(self.settings),
589
+ "faster_whisper_model": self.settings.faster_whisper_model,
590
+ "whisper_cpp_model": self.settings.whisper_cpp_model,
591
+ "whisper_cpp_binary": self.settings.whisper_cpp_binary,
592
+ "whisper_cpp_language": self.settings.whisper_cpp_language,
593
+ "model_files": model,
594
+ "ocr_enabled": self.settings.ocr_enabled,
595
+ "ocr_model_dir": self.settings.ocr_model_dir,
596
+ "ocr_files": ocr,
597
+ "languages": self.settings.transcript_languages,
598
+ "frame_budget": self.settings.frame_budget,
599
+ "implementation": _sha256_file(Path(__file__).with_name("runners.py")),
600
+ "extraction_implementation": extraction_implementation_fingerprint(),
601
+ }
602
+ runtime_versions: dict[str, str] = {}
603
+ for distribution in ("faster-whisper", "ctranslate2", "rapidocr", "onnxruntime", "Pillow"):
604
+ try:
605
+ runtime_versions[distribution] = version(distribution)
606
+ except PackageNotFoundError:
607
+ runtime_versions[distribution] = "not-installed"
608
+ binary = shutil.which(self.settings.whisper_cpp_binary) if self.settings.whisper_cpp_binary else None
609
+ payload["runtime_versions"] = runtime_versions
610
+ payload["whisper_cpp_binary_sha256"] = _sha256_file(Path(binary)) if binary is not None else None
611
+ return "sha256:" + hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
612
+
613
+ def _reusable_content(self, output_dir: Path, previous: Any, ref: ContentRef) -> ContentBundle | None:
614
+ """Reuse a verified platform stage without disguising it as a fixture."""
615
+ cache = getattr(previous, "content_cache", None)
616
+ if not isinstance(cache, dict) or cache.get("origin") != "platform":
617
+ return None
618
+ if cache.get("processing_fingerprint") != self._processing_fingerprint():
619
+ return None
620
+ if any(previous.stages.get(stage) != "complete" for stage in ("acquire", "extract")):
621
+ return None
622
+ try:
623
+ for filename in ("knowledge.json", "content.json", "source.json"):
624
+ path = output_dir / filename
625
+ if path.is_symlink() or _sha256_file(path) != cache.get(filename):
626
+ return None
627
+ bundle = ExtractionBundleV2.model_validate_json((output_dir / "knowledge.json").read_text(encoding="utf-8"))
628
+ if len(bundle.contents) != 1 or len(bundle.sources) != 1:
629
+ return None
630
+ content = bundle.contents[0]
631
+ if (
632
+ not self._same_content_unit(ref, content)
633
+ or previous.source_ids != [content.source.source_id]
634
+ or content.source_revision != previous.source_revision
635
+ or content.completeness.status != "complete"
636
+ ):
637
+ return None
638
+ for asset in content.assets:
639
+ path = output_dir / asset.relative_path
640
+ if (
641
+ not path.resolve().is_relative_to(output_dir.resolve())
642
+ or path.is_symlink() or path.stat().st_size != asset.byte_size
643
+ or _sha256_file(path) != asset.sha256
644
+ ):
645
+ return None
646
+ return content
647
+ except (OSError, ValueError, TypeError):
648
+ return None
649
+
650
+ def _acquired_content(self, ref: ContentRef, output_dir: Path, previous: Any, access: AccessContext,
651
+ *, refresh: bool, content_type: ContentType | None) -> tuple[ContentBundle, Any]:
652
+ from ytk.pipeline.acquisition_checkpoint import (
653
+ configuration_fingerprint,
654
+ load_checkpoint,
655
+ save_checkpoint,
656
+ )
657
+
658
+ store = self._store(output_dir)
659
+ configuration = configuration_fingerprint(self.settings, self._processing_fingerprint())
660
+ if store is not None and not refresh:
661
+ restored = load_checkpoint(store, previous, ref, configuration,
662
+ same_unit=self._same_content_unit, sanitize=_sanitize_metadata)
663
+ if restored is not None and (content_type is None or restored[0].source.content_type == content_type):
664
+ self._assert_output_ownership(output_dir, previous, restored[0].source)
665
+ return restored
666
+ try:
667
+ acquisition = self._acquire(ref, work_dir=output_dir / "artifacts" / "acquire", access=access, refresh=refresh)
668
+ except Exception as exc:
669
+ converted = self._convert_sidecar_error(exc, output_dir)
670
+ if converted is not None:
671
+ raise converted from exc
672
+ raise
673
+ acquired_ref = getattr(acquisition, "source", None)
674
+ if isinstance(acquired_ref, ContentRef):
675
+ ref = acquired_ref
676
+ self._assert_output_ownership(output_dir, previous, ref)
677
+ content = self._content_from_acquisition(ref, acquisition, output_dir)
678
+ if store is not None:
679
+ save_checkpoint(store, content, acquisition, configuration, sanitize=_sanitize_metadata)
680
+ return content, acquisition
681
+
682
+ @staticmethod
683
+ def _same_content_unit(ref: ContentRef, content: ContentBundle) -> bool:
684
+ if ref.source_id == content.source.source_id:
685
+ return True
686
+ if ref.platform != "bilibili" or ref.variant_id is not None:
687
+ return False
688
+ if ref.platform != content.source.platform or ref.content_id != content.source.content_id:
689
+ return False
690
+ from urllib.parse import parse_qs, urlsplit
691
+
692
+ try:
693
+ requested_part = int(ref.display.get("p") or 1)
694
+ saved_part = content.metadata.get("p") or content.source.display.get("p")
695
+ if saved_part is None:
696
+ saved_part = parse_qs(urlsplit(content.source.canonical_url).query).get("p", [1])[0]
697
+ return requested_part == int(str(saved_part))
698
+ except (ValueError, TypeError):
699
+ return False
700
+
701
+ def _content_from_fixture(self, fixture: FixtureContent, output_dir: Path) -> ContentBundle:
702
+ source_id = fixture.source.source_id
703
+ blocks = list(fixture.blocks)
704
+ if fixture.subtitle_text:
705
+ blocks.extend(self._blocks_from_subtitle_text(fixture.subtitle_text, source_id, start_order=len(blocks)))
706
+ assets = self._copy_fixture_media(fixture.media, source_id, output_dir)
707
+ fingerprint = {
708
+ "blocks": [block.block_id for block in blocks],
709
+ "assets": [asset.sha256 for asset in assets],
710
+ }
711
+ return ContentBundle(
712
+ source=fixture.source,
713
+ source_revision=compute_source_revision(source_id, fingerprint),
714
+ assets=assets,
715
+ blocks=blocks,
716
+ completeness=fixture.completeness,
717
+ provider_trace=fixture.provider_trace,
718
+ metadata=dict(fixture.metadata),
719
+ )
720
+
721
+ def _browser_content_from_fixture(self, fixture: FixtureContent, output_dir: Path, url: str) -> tuple[ContentBundle, Any]:
722
+ """Reject bad capture descriptors before touching a previous delivery."""
723
+ from ytk.ingestion.browser_acquired import verify_browser_acquisition
724
+ if len(fixture.media) != 1:
725
+ raise UserFacingError("browser acquisition requires one actual MP4 video")
726
+ local = fixture.media[0].get("local_path")
727
+ if (not isinstance(local, str) or Path(local).suffix.lower() != ".mp4"
728
+ or Path(local).is_symlink() or fixture.media[0].get("media_type") != "video"):
729
+ raise UserFacingError("browser acquisition requires a regular local MP4 video")
730
+ with tempfile.TemporaryDirectory(prefix=".kd-browser-verify-", dir=output_dir.parent) as work:
731
+ stage = Path(work)
732
+ content = self._content_from_fixture(fixture, stage)
733
+ content, acquisition = verify_browser_acquisition(content, stage, url)
734
+ asset = content.assets[0]
735
+ relative = f"assets/browser-{asset.sha256}.mp4"
736
+ target = output_dir / relative
737
+ target.parent.mkdir(parents=True, exist_ok=True)
738
+ _copy_current_asset(stage / asset.relative_path, target)
739
+ published = asset.model_copy(update={"relative_path": relative})
740
+ content = content.model_copy(update={"assets": [published]})
741
+ acquisition = acquisition.model_copy(update={"media": [acquisition.media[0].model_copy(update={"local_path": str(target)})]})
742
+ return content, acquisition
743
+
744
+ def _content_from_acquisition(self, ref: ContentRef, acquisition: Any, output_dir: Path) -> ContentBundle:
745
+ source_id = ref.source_id
746
+ media = list(getattr(acquisition, "media", []) or [])
747
+ assets = list(getattr(acquisition, "assets", []) or [])
748
+ if not assets and media:
749
+ assets = self._materialize_media(media, source_id, output_dir)
750
+ blocks = self._blocks_from_acquisition(acquisition, source_id)
751
+ completeness = getattr(acquisition, "completeness", None) or ContentCompleteness(
752
+ status="unknown", reason_code=getattr(acquisition, "reason_code", None)
753
+ )
754
+ fingerprint = {
755
+ "blocks": [block.block_id for block in blocks],
756
+ "assets": [asset.sha256 for asset in assets],
757
+ "metadata_title": str(getattr(acquisition, "metadata", {}).get("title", "")),
758
+ }
759
+ return ContentBundle(
760
+ source=ref,
761
+ source_revision=compute_source_revision(source_id, fingerprint),
762
+ assets=assets,
763
+ blocks=blocks,
764
+ completeness=completeness,
765
+ provider_trace=list(getattr(acquisition, "provider_trace", []) or []),
766
+ metadata=dict(getattr(acquisition, "metadata", {}) or {}),
767
+ )
768
+
769
+ @staticmethod
770
+ def _blocks_from_acquisition(acquisition: Any, source_id: str) -> list[ContentBlock]:
771
+ """Turn adapter ``AcquiredBlock`` items into locator-keyed content blocks."""
772
+ blocks: list[ContentBlock] = []
773
+ seen: set[str] = set()
774
+ for raw in getattr(acquisition, "blocks", []) or []:
775
+ locator = raw.locator
776
+ raw_text = raw.raw_text
777
+ block_id = make_block_id(source_id, locator, raw_text)
778
+ if block_id in seen:
779
+ continue
780
+ seen.add(block_id)
781
+ blocks.append(
782
+ ContentBlock(
783
+ block_id=block_id,
784
+ source_id=source_id,
785
+ kind=raw.kind,
786
+ raw_text=raw_text,
787
+ cleaned_text=getattr(raw, "cleaned_text", None),
788
+ language=getattr(raw, "language", None),
789
+ order=int(getattr(raw, "order", len(blocks))),
790
+ locator=locator,
791
+ )
792
+ )
793
+ return blocks
794
+
795
+ def _copy_fixture_media(
796
+ self, media: Sequence[Mapping[str, object]], source_id: str, output_dir: Path
797
+ ) -> list[MediaAsset]:
798
+ assets: list[MediaAsset] = []
799
+ assets_dir = output_dir / "assets"
800
+ for index, entry in enumerate(media):
801
+ local = entry.get("local_path")
802
+ if not isinstance(local, str):
803
+ continue
804
+ source_path = Path(local)
805
+ if not source_path.is_file():
806
+ raise UserFacingError(f"fixture media not found: {source_path}")
807
+ assets_dir.mkdir(parents=True, exist_ok=True)
808
+ media_type = str(entry.get("media_type") or "image")
809
+ suffix = source_path.suffix or ".bin"
810
+ relative = f"assets/{media_type}-{index:03d}{suffix}"
811
+ target = output_dir / relative
812
+ _copy_current_asset(source_path, target)
813
+ assets.append(
814
+ MediaAsset(
815
+ asset_id=f"asset-{index:04d}",
816
+ source_id=source_id,
817
+ relative_path=relative,
818
+ media_type=media_type, # type: ignore[arg-type]
819
+ order=int(entry.get("order", index)), # type: ignore[call-overload]
820
+ byte_size=target.stat().st_size,
821
+ sha256=_sha256_file(target),
822
+ )
823
+ )
824
+ return assets
825
+
826
+ def _materialize_media(
827
+ self, media: Sequence[Any], source_id: str, output_dir: Path
828
+ ) -> list[MediaAsset]:
829
+ """Download/attach media descriptors produced by an adapter."""
830
+ assets: list[MediaAsset] = []
831
+ assets_dir = output_dir / "assets"
832
+ for index, descriptor in enumerate(media):
833
+ local_path = getattr(descriptor, "local_path", None)
834
+ if not local_path:
835
+ continue
836
+ source_path = Path(local_path)
837
+ if not source_path.is_file():
838
+ continue
839
+ assets_dir.mkdir(parents=True, exist_ok=True)
840
+ media_type = getattr(descriptor, "media_type", "image")
841
+ relative = f"assets/{media_type}-{index:03d}{source_path.suffix or '.bin'}"
842
+ target = output_dir / relative
843
+ _copy_current_asset(source_path, target)
844
+ assets.append(
845
+ MediaAsset(
846
+ asset_id=f"asset-{index:04d}",
847
+ source_id=source_id,
848
+ relative_path=relative,
849
+ media_type=cast(Literal["video", "audio", "image", "frame"], media_type),
850
+ order=int(getattr(descriptor, "order", index)),
851
+ byte_size=target.stat().st_size,
852
+ sha256=_sha256_file(target),
853
+ )
854
+ )
855
+ return assets
856
+
857
+ def _blocks_from_subtitle_text(self, text: str, source_id: str, *, start_order: int = 0) -> list[ContentBlock]:
858
+ from ytk.ingestion.adapters.base import SubtitleDescriptor
859
+ from ytk.ingestion.subtitles import parse_subtitle
860
+ from ytk.models.content import ParagraphLocator
861
+
862
+ if "-->" in text or text.lstrip().startswith("WEBVTT"):
863
+ parsed = parse_subtitle(
864
+ SubtitleDescriptor(kind="inline", format="vtt" if text.lstrip().startswith("WEBVTT") else "srt", inline_data=text),
865
+ source_id=source_id,
866
+ )
867
+ return [ContentBlock(
868
+ block_id=make_block_id(source_id, block.locator, block.raw_text),
869
+ source_id=source_id, kind="caption", raw_text=block.raw_text,
870
+ order=start_order + index, locator=block.locator,
871
+ ) for index, block in enumerate(parsed.blocks)]
872
+
873
+ blocks: list[ContentBlock] = []
874
+ for index, line in enumerate(text.splitlines()):
875
+ stripped = line.strip()
876
+ if not stripped:
877
+ continue
878
+ locator = ParagraphLocator(paragraph=index)
879
+ blocks.append(
880
+ ContentBlock(
881
+ block_id=make_block_id(source_id, locator, stripped),
882
+ source_id=source_id,
883
+ kind="caption",
884
+ raw_text=stripped,
885
+ order=start_order + index,
886
+ locator=locator,
887
+ )
888
+ )
889
+ return blocks
890
+
891
+ def _maybe_frame_ocr(self, content: ContentBundle, output_dir: Path) -> ContentBundle:
892
+ """Budget-bound OCR of sampled video frames for videos without narration (T32.2).
893
+
894
+ Only runs when the engine is enabled, a local video asset exists and no
895
+ caption/transcript timed blocks exist (narration evidence wins).
896
+ """
897
+ if not bool(getattr(self.settings, "ocr_enabled", False)):
898
+ return content
899
+ max_frames = self.settings.frame_budget
900
+ if max_frames <= 0:
901
+ return content
902
+ if self.ocr_runner is None:
903
+ return content
904
+ video_assets = [asset for asset in content.assets if asset.media_type == "video"]
905
+ if not video_assets:
906
+ return content
907
+ has_narration = any(
908
+ block.kind in {"caption", "transcript"} and block.locator.kind == "time_range"
909
+ for block in content.blocks
910
+ )
911
+ if has_narration:
912
+ return content
913
+ try:
914
+ from ytk.ingestion.media import MediaService
915
+ from ytk.ocr.engine import get_engine
916
+ from ytk.ocr.pipeline import run_ocr_for_frames
917
+
918
+ service = MediaService(settings=self.settings)
919
+ source_path = Path(output_dir) / video_assets[0].relative_path
920
+ if not source_path.is_file():
921
+ return content
922
+ frames_dir = Path(output_dir) / "artifacts" / "frames"
923
+ budget = {"max_frames": max_frames}
924
+ frames = service.sample_frames(source_path, frames_dir, budget=budget)
925
+ if not frames:
926
+ return content
927
+ frame_refs = [(float(frame.time), frame.path) for frame in frames]
928
+ start_order = max((block.order for block in content.blocks), default=-1) + 1
929
+ blocks = run_ocr_for_frames(
930
+ frame_refs,
931
+ source_id=content.source.source_id,
932
+ start_order=start_order,
933
+ frame_budget=budget,
934
+ engine=get_engine(model_dir=str(self.settings.resolved_ocr_model_dir())),
935
+ )
936
+ except ModelUnavailableError:
937
+ raise
938
+ except Exception as exc: # frame OCR is enhancement: degrade, never crash
939
+ LOGGER.warning("video frame OCR failed for %s: %s", content.source.source_id, exc)
940
+ return content
941
+ if not blocks:
942
+ return content
943
+ existing = {block.block_id for block in content.blocks}
944
+ merged = list(content.blocks) + [block for block in blocks if block.block_id not in existing]
945
+ return content.model_copy(update={"blocks": merged})
946
+
947
+ # ------------------------------------------------------------ enrichment
948
+ def _maybe_ocr(self, content: ContentBundle, output_dir: Path) -> ContentBundle:
949
+ if content.source.content_type != "image_note":
950
+ return content
951
+ if self.ocr_runner is None:
952
+ image_count = sum(asset.media_type == "image" for asset in content.assets)
953
+ recognized = {
954
+ block.locator.image_index for block in content.blocks
955
+ if block.kind == "ocr" and block.locator.kind == "image_region"
956
+ }
957
+ if not set(range(1, image_count + 1)) <= recognized:
958
+ return content.model_copy(update={"completeness": ContentCompleteness(
959
+ status="partial", checked=list(content.completeness.checked),
960
+ missing=list(dict.fromkeys([*content.completeness.missing, "ocr"])),
961
+ reason_code="OCR_DISABLED" if not self.settings.ocr_enabled else "OCR_UNAVAILABLE",
962
+ )})
963
+ return content
964
+ try:
965
+ blocks, ocr_metadata = _split_runner_output(self.ocr_runner(content, output_dir))
966
+ except Exception as exc: # OCR unavailable/one image failed → PARTIAL, never crash
967
+ LOGGER.warning("OCR stage failed for %s: %s", content.source.source_id, exc)
968
+ return content.model_copy(
969
+ update={
970
+ "completeness": ContentCompleteness(
971
+ status="partial",
972
+ checked=list(content.completeness.checked) + ["ocr"],
973
+ missing=["ocr"],
974
+ reason_code="OCR_UNAVAILABLE",
975
+ )
976
+ }
977
+ )
978
+ existing = {block.block_id for block in content.blocks}
979
+ merged = list(content.blocks) + [block for block in blocks if block.block_id not in existing]
980
+ metadata = dict(content.metadata)
981
+ completeness = content.completeness
982
+ if ocr_metadata:
983
+ metadata["ocr"] = ocr_metadata
984
+ if not ocr_metadata.get("complete", True):
985
+ completeness = ContentCompleteness(
986
+ status="partial",
987
+ checked=list(content.completeness.checked) + ["ocr"],
988
+ missing=list(dict.fromkeys([*content.completeness.missing, "ocr"])),
989
+ reason_code="OCR_INCOMPLETE",
990
+ )
991
+ return content.model_copy(update={"blocks": merged, "metadata": metadata, "completeness": completeness})
992
+
993
+ def _maybe_asr(self, content: ContentBundle, output_dir: Path) -> ContentBundle:
994
+ if self.asr_runner is None:
995
+ return content
996
+ probe = content.metadata.get("media_probe")
997
+ if content.metadata.get("acquisition_mode") == "user_import" and isinstance(probe, dict) and probe.get("audio_streams") == 0:
998
+ return content
999
+ has_timed = any(
1000
+ block.locator.kind == "time_range" and block.kind in {"caption", "transcript"}
1001
+ for block in content.blocks
1002
+ )
1003
+ has_audio = any(asset.media_type in {"audio", "video"} for asset in content.assets)
1004
+ if has_timed or not has_audio:
1005
+ return content
1006
+ try:
1007
+ asr_output = self.asr_runner(content, output_dir)
1008
+ except ServiceNotConfiguredError:
1009
+ raise
1010
+ except ModelUnavailableError:
1011
+ raise
1012
+ except Exception as exc:
1013
+ LOGGER.warning("ASR stage failed for %s: %s", content.source.source_id, exc)
1014
+ return content.model_copy(
1015
+ update={
1016
+ "completeness": ContentCompleteness(
1017
+ status="partial",
1018
+ checked=list(content.completeness.checked) + ["asr"],
1019
+ missing=["asr"],
1020
+ reason_code="ASR_UNAVAILABLE",
1021
+ )
1022
+ }
1023
+ )
1024
+ blocks, asr_metadata = _split_runner_output(asr_output)
1025
+ existing = {block.block_id for block in content.blocks}
1026
+ merged = list(content.blocks) + [block for block in blocks if block.block_id not in existing]
1027
+ metadata = dict(content.metadata)
1028
+ if asr_metadata:
1029
+ metadata["asr"] = asr_metadata
1030
+ metadata["transcript_source"] = "asr"
1031
+ return content.model_copy(update={"blocks": merged, "metadata": metadata})
1032
+
1033
+ @staticmethod
1034
+ def _assess_completeness(acquisition: Any, content: ContentBundle) -> ContentBundle:
1035
+ try:
1036
+ from ytk.quality.completeness import assess
1037
+
1038
+ assessment = assess(acquisition, content)
1039
+ except Exception as exc: # assessment must never crash the run
1040
+ LOGGER.warning("completeness assessment failed: %s", exc)
1041
+ return content
1042
+ if assessment.content == content.completeness:
1043
+ return content
1044
+ return content.model_copy(update={"completeness": assessment.content})
1045
+
1046
+ # ------------------------------------------------------------ assembly
1047
+ def _assemble(self, ref: ContentRef, content: ContentBundle) -> ExtractionBundleV2:
1048
+ source_id = ref.source_id
1049
+ cleaned = self.cleaning.clean_blocks(content.blocks)
1050
+ ordered = sorted(cleaned, key=lambda block: block.order)
1051
+ chapters = content.metadata.get("chapters")
1052
+ chapter_list = chapters if isinstance(chapters, list) else None
1053
+ topics = self.segmentation.segment_blocks(ordered, chapters=chapter_list)
1054
+ extraction: BlockExtraction = self.extraction.extract_blocks(
1055
+ ordered,
1056
+ source_id=source_id,
1057
+ source_revision=content.source_revision,
1058
+ topic_groups=topics,
1059
+ block_processors=self._block_processors(ordered, content),
1060
+ )
1061
+ metadata = dict(content.metadata)
1062
+ metadata.setdefault("title", ref.display.get("title") or source_id)
1063
+ metadata.setdefault("canonical_url", ref.canonical_url)
1064
+ metadata.setdefault("platform", ref.platform)
1065
+ metadata.setdefault("source_id", source_id)
1066
+ atoms = list(extraction.atoms)
1067
+ evidence = dict(extraction.evidence)
1068
+ if not atoms and self.coarse_atoms:
1069
+ atoms, coarse_evidence = self._coarse_atoms(
1070
+ ordered, topics, evidence, source_id=source_id,
1071
+ source_revision=content.source_revision,
1072
+ block_processors=self._block_processors(ordered, content),
1073
+ )
1074
+ evidence.update(coarse_evidence)
1075
+ if atoms:
1076
+ metadata = {**metadata, "coarse_fallback": "segment_summary"}
1077
+ LOGGER.warning(
1078
+ "no structured atoms; emitted %d coarse segment summaries "
1079
+ "(confidence 0.25, explicitly labelled)", len(atoms)
1080
+ )
1081
+ return ExtractionBundleV2(
1082
+ sources=[ref],
1083
+ contents=[content.model_copy(update={"blocks": ordered})],
1084
+ evidence_index=evidence,
1085
+ topics=list(topics),
1086
+ knowledge=atoms,
1087
+ metadata=metadata,
1088
+ skill_spec=None,
1089
+ )
1090
+
1091
+ @staticmethod
1092
+ def _coarse_atoms(
1093
+ blocks: Sequence[ContentBlock],
1094
+ topics: Sequence[Any],
1095
+ evidence: dict[str, EvidenceRefV2],
1096
+ *,
1097
+ source_id: str,
1098
+ source_revision: str,
1099
+ block_processors: Mapping[str, str],
1100
+ ) -> tuple[list[KnowledgeAtomV2], dict[str, EvidenceRefV2]]:
1101
+ """每个 topic 一条低置信分段摘要。
1102
+
1103
+ 只在显式开启 coarse 模式且完全抽不出原子时调用。产物带
1104
+ ``metadata.coarse=True`` 与 0.25 置信度,``explanation`` 明写它是
1105
+ 分段摘要而不是抽取到的知识——允许"弱结论",不允许"假结论"。
1106
+ """
1107
+ # 证据必须来自 block 本身:即使走粗粒度兜底,也不允许出现无定位的原子。
1108
+ by_block: dict[str, list[str]] = {}
1109
+ for ev_id, ref in evidence.items():
1110
+ by_block.setdefault(getattr(ref, "block_id", ""), []).append(ev_id)
1111
+ text_of = {b.block_id: (b.raw_text or "").strip() for b in blocks}
1112
+ block_by_id = {b.block_id: b for b in blocks}
1113
+ new_evidence: dict[str, EvidenceRefV2] = {}
1114
+
1115
+ def ensure_evidence(block_id: str) -> list[str]:
1116
+ existing = by_block.get(block_id)
1117
+ if existing:
1118
+ return existing
1119
+ block = block_by_id.get(block_id)
1120
+ if block is None:
1121
+ return []
1122
+ ev_id = "%s@%s#%s#%04d" % (source_id, source_revision, block_id,
1123
+ len(new_evidence) + 1)
1124
+ # 粗粒度兜底拿不准来源时,标记为 unknown 而不是谎称 subtitle_auto
1125
+ processor = (block_processors or {}).get(block.block_id) or "unknown"
1126
+ new_evidence[ev_id] = EvidenceRefV2(
1127
+ evidence_id=ev_id,
1128
+ source_id=source_id,
1129
+ source_revision=source_revision,
1130
+ block_id=block.block_id,
1131
+ quote=(block.raw_text or "").strip()[:500],
1132
+ locator=block.locator,
1133
+ processor=processor, # type: ignore[arg-type]
1134
+ )
1135
+ by_block.setdefault(block_id, []).append(ev_id)
1136
+ return [ev_id]
1137
+
1138
+ atoms: list[KnowledgeAtomV2] = []
1139
+ for index, topic in enumerate(topics, start=1):
1140
+ block_ids = [b for b in getattr(topic, "block_ids", []) if b in text_of]
1141
+ joined = " ".join(text_of[b] for b in block_ids).strip()
1142
+ ev_ids = [e for b in block_ids for e in ensure_evidence(b)]
1143
+ if not joined or not ev_ids:
1144
+ continue
1145
+ title = getattr(topic, "title", "") or "segment %d" % index
1146
+ atoms.append(
1147
+ KnowledgeAtomV2(
1148
+ atom_id="coarse-%d" % index,
1149
+ atom_type="segment_summary",
1150
+ content=joined[:400],
1151
+ explanation=(
1152
+ "低置信分段摘要:本段未抽取出结构化知识,这里只是按主题切出的"
1153
+ "原文摘录,不能当作结论使用。"
1154
+ ),
1155
+ source_ids=[source_id],
1156
+ evidence_ids=ev_ids,
1157
+ confidence=0.25,
1158
+ topic_path=[title],
1159
+ metadata={"coarse": True, "block_count": len(block_ids)},
1160
+ )
1161
+ )
1162
+ return atoms, new_evidence
1163
+
1164
+ @staticmethod
1165
+ def _block_processors(blocks: Sequence[ContentBlock], content: ContentBundle) -> dict[str, str]:
1166
+ """Truthful evidence processors: ASR text is never stamped as a subtitle."""
1167
+ subtitle_meta = content.metadata.get("subtitle")
1168
+ if not isinstance(subtitle_meta, dict):
1169
+ subtitle_meta = {}
1170
+ transcript_source = content.metadata.get("transcript_source")
1171
+ track_kind = subtitle_meta.get("kind")
1172
+ track_format = subtitle_meta.get("format")
1173
+
1174
+ def caption_processor() -> str:
1175
+ if track_kind == "manual":
1176
+ return "subtitle_manual"
1177
+ return {
1178
+ "json3": "subtitle_json3",
1179
+ "srt": "subtitle_srt",
1180
+ "vtt": "subtitle_vtt",
1181
+ }.get(str(track_format or ""), "subtitle_auto")
1182
+
1183
+ processors: dict[str, str] = {}
1184
+ for block in blocks:
1185
+ if content.metadata.get("acquisition_mode") == "user_import" and block.locator.kind == "video_frame":
1186
+ processors[block.block_id] = "visual_review"
1187
+ elif content.metadata.get("transcript_source") == "user_provided" and block.kind == "caption":
1188
+ processors[block.block_id] = "user_provided_subtitle"
1189
+ elif block.kind == "ocr":
1190
+ processors[block.block_id] = "ocr"
1191
+ elif block.kind in {"paragraph", "title", "description"}:
1192
+ processors[block.block_id] = "author_text"
1193
+ elif transcript_source == "asr":
1194
+ processors[block.block_id] = "asr"
1195
+ else:
1196
+ processors[block.block_id] = caption_processor()
1197
+ return processors
1198
+
1199
+ # ------------------------------------------------------------- outcomes
1200
+ def _resolve_formats(self, ref: ContentRef, formats: Sequence[str] | None) -> set[str]:
1201
+ if formats is None:
1202
+ registration = self.registry.get(ref.platform)
1203
+ return set(registration.default_formats)
1204
+ return {value.strip() for value in formats if value and value.strip()}
1205
+
1206
+ def _decide_outcomes(
1207
+ self, bundle: ExtractionBundleV2, requested: set[str]
1208
+ ) -> tuple[Outcome, dict[str, str], dict[str, str], list[str]]:
1209
+ reasons: list[str] = []
1210
+ format_status: dict[str, str] = {}
1211
+ format_reasons: dict[str, str] = {}
1212
+ completeness = bundle.contents[0].completeness if bundle.contents else None
1213
+ content_is_complete = completeness is not None and completeness.status == "complete"
1214
+ has_knowledge = bool(bundle.knowledge)
1215
+ has_steps = any(
1216
+ atom.atom_type in {"workflow_step", "terminal_command", "configuration_snippet", "command_sequence", "code_block"}
1217
+ for atom in bundle.knowledge
1218
+ )
1219
+
1220
+ if not has_knowledge:
1221
+ # NO_KNOWLEDGE only when content is provably complete (plan §4.5).
1222
+ if content_is_complete and _no_knowledge_gate(bundle, completeness):
1223
+ for fmt in requested:
1224
+ format_status[fmt] = "pending" if fmt in {"json", "md", "summary"} else "not_applicable"
1225
+ if format_status[fmt] == "not_applicable":
1226
+ format_reasons[fmt] = "NO_KNOWLEDGE"
1227
+ reasons.append("NO_KNOWLEDGE: content complete but no knowledge atoms extracted")
1228
+ return OUTCOME_NO_KNOWLEDGE, format_status, format_reasons, reasons
1229
+ for fmt in requested:
1230
+ format_status[fmt] = "blocked"
1231
+ format_reasons[fmt] = "CONTENT_INCOMPLETE"
1232
+ reasons.append(
1233
+ f"BLOCKED: content incomplete ({completeness.status if completeness else 'unknown'}) "
1234
+ "and no knowledge atoms were extracted"
1235
+ )
1236
+ return OUTCOME_BLOCKED, format_status, format_reasons, reasons
1237
+
1238
+ for fmt in requested:
1239
+ if fmt == "skill" and not has_steps:
1240
+ format_status[fmt] = "not_applicable"
1241
+ format_reasons[fmt] = "OUTPUTS_NOT_APPLICABLE"
1242
+ reasons.append("skill NOT_APPLICABLE: knowledge has no actionable steps")
1243
+ else:
1244
+ format_status[fmt] = "pending"
1245
+ actionable = [fmt for fmt in requested if format_status.get(fmt) == "pending"]
1246
+ if requested and not actionable:
1247
+ reasons.append("FAILED: every requested format is not applicable (OUTPUTS_NOT_APPLICABLE)")
1248
+ return OUTCOME_FAILED, format_status, format_reasons, reasons
1249
+ if completeness is not None and completeness.status != "complete":
1250
+ reasons.append(f"PARTIAL: content completeness={completeness.status}")
1251
+ return OUTCOME_PARTIAL, format_status, format_reasons, reasons
1252
+ return OUTCOME_COMPLETE, format_status, format_reasons, reasons
1253
+
1254
+ # --------------------------------------------------------------- render
1255
+ def _render(
1256
+ self,
1257
+ bundle: ExtractionBundleV2,
1258
+ requested: set[str],
1259
+ output_dir: Path,
1260
+ format_status: dict[str, str],
1261
+ *,
1262
+ doc_language: str | None,
1263
+ ) -> dict[str, Path]:
1264
+ written: dict[str, Path] = {}
1265
+ # The canonical v2 product (schema_version=2 JSON) is owned by the
1266
+ # pipeline, not by a pluggable renderer: it is the artifact every other
1267
+ # output and the acceptance validator read back.
1268
+ canonical = self._write_canonical_artifacts(bundle, output_dir)
1269
+ for fmt in sorted(requested):
1270
+ if format_status.get(fmt) in {"not_applicable", "blocked"}:
1271
+ continue
1272
+ if fmt == "json":
1273
+ format_status[fmt] = "complete"
1274
+ written[fmt] = canonical["knowledge"]
1275
+ continue
1276
+ renderer = self.renderers.get(fmt)
1277
+ if renderer is None:
1278
+ format_status[fmt] = "failed"
1279
+ continue
1280
+ target = output_dir / _default_filename(fmt)
1281
+ try:
1282
+ render_kwargs: dict[str, Any] = {"doc_language": doc_language}
1283
+ if fmt == "doc":
1284
+ render_kwargs["allow_cloud"] = self._cloud_text_allowed(bundle)
1285
+ render_kwargs["settings"] = self.settings
1286
+ (output_dir / ".knowledge-doc-status.json").unlink(missing_ok=True)
1287
+ renderer(bundle, target, **render_kwargs)
1288
+ except Exception as exc: # one format failing must not kill the others
1289
+ LOGGER.warning("renderer %s failed: %s", fmt, exc)
1290
+ format_status[fmt] = "failed"
1291
+ continue
1292
+ if fmt == "doc":
1293
+ from ytk.renderers.knowledge_doc_renderer import complete_doc_status
1294
+
1295
+ if not complete_doc_status(output_dir):
1296
+ format_status[fmt] = "blocked"
1297
+ try:
1298
+ status = json.loads((output_dir / ".knowledge-doc-status.json").read_text(encoding="utf-8"))
1299
+ filename = status.get("output_file", "")
1300
+ candidate = output_dir / filename
1301
+ if (
1302
+ isinstance(filename, str) and filename and Path(filename).name == filename
1303
+ and candidate.is_file() and _sha256_file(candidate) == status.get("output_sha256")
1304
+ ):
1305
+ written[fmt] = candidate
1306
+ except (OSError, ValueError, TypeError):
1307
+ pass
1308
+ continue
1309
+ format_status[fmt] = "complete"
1310
+ written[fmt] = target
1311
+ return written
1312
+
1313
+ def _write_canonical_artifacts(self, bundle: ExtractionBundleV2, output_dir: Path) -> dict[str, Path]:
1314
+ """Write schema-v2 JSON artifacts atomically (temp file -> replace)."""
1315
+ knowledge_payload = dump_bundle_json(bundle)
1316
+ source_payload = {
1317
+ "schema_version": 2,
1318
+ "sources": [ref.public_dict() for ref in bundle.sources],
1319
+ "metadata": bundle.metadata,
1320
+ }
1321
+ content_payload = {
1322
+ "schema_version": 2,
1323
+ "contents": [item.model_dump(mode="json") for item in bundle.contents],
1324
+ }
1325
+ store = self._store(output_dir)
1326
+ if store is not None:
1327
+ return {
1328
+ "knowledge": store.write_json("knowledge.json", knowledge_payload),
1329
+ "source": store.write_json("source.json", source_payload),
1330
+ "content": store.write_json("content.json", content_payload),
1331
+ }
1332
+ knowledge = output_dir / "knowledge.json"
1333
+ _atomic_write_json(knowledge, knowledge_payload)
1334
+ source = output_dir / "source.json"
1335
+ _atomic_write_json(source, source_payload)
1336
+ content = output_dir / "content.json"
1337
+ _atomic_write_json(content, content_payload)
1338
+ return {"knowledge": knowledge, "source": source, "content": content}
1339
+
1340
+ def _store(self, output_dir: Path) -> Any | None:
1341
+ factory = self.artifact_store_factory
1342
+ if factory is None:
1343
+ return None
1344
+ try:
1345
+ return factory(output_dir)
1346
+ except Exception as exc: # store is an optimization, never a blocker
1347
+ LOGGER.warning("artifact store unavailable for %s: %s", output_dir, exc)
1348
+ return None
1349
+
1350
+ def _read_previous_manifest(self, output_dir: Path) -> Any | None:
1351
+ store = self._store(output_dir)
1352
+ if store is None:
1353
+ return None
1354
+ try:
1355
+ return store.read_manifest()
1356
+ except Exception as exc: # pragma: no cover - defensive
1357
+ LOGGER.warning("existing manifest unreadable (%s); treating as absent", exc)
1358
+ return None
1359
+
1360
+ @staticmethod
1361
+ def _assert_output_ownership(
1362
+ output_dir: Path, previous: Any | None, ref: ContentRef, *, allow_unresolved_variant: bool = False
1363
+ ) -> None:
1364
+ """Refuse to mix two different works into one output directory (A39/A02/A03).
1365
+
1366
+ Identity comparison is two-tiered: the base (platform:content_id) must
1367
+ match, AND — because a Bilibili ``cid`` denotes a different content unit —
1368
+ the platform variant must match when the existing manifest carries one.
1369
+ A pre-acquire ref without a variant is refused when the directory already
1370
+ holds a variant-bearing record (ambiguous ownership, fail closed).
1371
+ """
1372
+ if previous is None:
1373
+ return
1374
+ existing = set(getattr(previous, "source_ids", []) or [])
1375
+ if not existing:
1376
+ return
1377
+
1378
+ def _split(source_id: str) -> tuple[str, str | None]:
1379
+ parts = source_id.split(":")
1380
+ return ":".join(parts[:2]), (parts[2] if len(parts) > 2 else None)
1381
+
1382
+ if _split(ref.source_id)[0] not in {_split(item)[0] for item in existing}:
1383
+ raise UserFacingError(
1384
+ f"output directory {output_dir} already belongs to {sorted(existing)}; "
1385
+ f"refusing to write {ref.source_id} into it (use a per-source --out)"
1386
+ )
1387
+ existing_variants = {variant for _, variant in (_split(item) for item in existing) if variant}
1388
+ new_variant = _split(ref.source_id)[1]
1389
+ if allow_unresolved_variant and ref.platform == "bilibili" and new_variant is None:
1390
+ return # cid is checked after acquisition, before any canonical/media write
1391
+ if existing_variants and (new_variant is None or new_variant not in existing_variants):
1392
+ raise UserFacingError(
1393
+ f"output directory {output_dir} already holds variant(s) {sorted(existing_variants)} of "
1394
+ f"the same work; refusing to overwrite with {ref.source_id} "
1395
+ f"(different content unit — use a per-part --out)"
1396
+ )
1397
+
1398
+ @staticmethod
1399
+ def _assert_no_unresolved_remote(previous: Any | None) -> None:
1400
+ """Never re-dispatch a task whose remote submission state is unknown (A36)."""
1401
+ if previous is None:
1402
+ return
1403
+ unresolved: dict[str, dict[str, Any]] = {}
1404
+ try:
1405
+ from ytk.pipeline.state import unresolved_remote_tasks
1406
+
1407
+ unresolved = dict(unresolved_remote_tasks(previous))
1408
+ except Exception: # pragma: no cover - defensive
1409
+ return
1410
+ unresolved = {
1411
+ provider: record for provider, record in unresolved.items()
1412
+ if not (
1413
+ provider == "dtk" and record.get("task_id")
1414
+ and all(source_id.startswith("douyin:") for source_id in previous.source_ids)
1415
+ and previous.source_ids
1416
+ )
1417
+ }
1418
+ if unresolved:
1419
+ from ytk.errors import AnalysisBlockedError
1420
+
1421
+ raise AnalysisBlockedError(
1422
+ "REMOTE_SUBMISSION_UNKNOWN: this task has an unresolved remote submission "
1423
+ f"({sorted(unresolved)}); refusing to re-dispatch. Resolve or clear it "
1424
+ "explicitly before refreshing."
1425
+ )
1426
+
1427
+ @staticmethod
1428
+ def _remove_stale_artifacts(previous: Any | None, written: Mapping[str, Path], output_dir: Path) -> None:
1429
+ """Delete artifacts recorded by an older manifest that this run superseded (R-12)."""
1430
+ if previous is None:
1431
+ return
1432
+ keep = {path.resolve() for path in written.values()}
1433
+ for fmt, entry in (getattr(previous, "outputs", {}) or {}).items():
1434
+ if fmt == "doc" and fmt in written:
1435
+ continue
1436
+ recorded = getattr(entry, "path", None)
1437
+ if not recorded:
1438
+ continue
1439
+ candidate = (output_dir / recorded).resolve()
1440
+ if candidate in keep or not candidate.is_file():
1441
+ continue
1442
+ try:
1443
+ if candidate.is_relative_to(output_dir.resolve()):
1444
+ candidate.unlink()
1445
+ LOGGER.info("removed stale artifact %s (was %s)", candidate.name, fmt)
1446
+ except OSError as exc: # pragma: no cover - defensive
1447
+ LOGGER.warning("could not remove stale artifact %s: %s", candidate, exc)
1448
+
1449
+ def _sanitize_bundle(self, bundle: ExtractionBundleV2) -> ExtractionBundleV2:
1450
+ """Strip credentials/access tokens from everything that will be persisted (A24).
1451
+
1452
+ Redaction is two-layered (adversarial review P0): key-name based — any
1453
+ metadata key that names a credential (SESSDATA, signature, xsec_token …)
1454
+ loses its value — and value-shape based via :func:`ytk.logging.redact_payload`
1455
+ for URL/token patterns anywhere else.
1456
+ """
1457
+ sanitized_metadata = _sanitize_metadata(dict(bundle.metadata or {}))
1458
+ contents = [
1459
+ content.model_copy(update={"metadata": _sanitize_metadata(dict(content.metadata or {}))})
1460
+ for content in bundle.contents
1461
+ ]
1462
+ return bundle.model_copy(
1463
+ update={
1464
+ "metadata": sanitized_metadata if isinstance(sanitized_metadata, dict) else {},
1465
+ "contents": contents,
1466
+ }
1467
+ )
1468
+
1469
+ @staticmethod
1470
+ def _adjust_outcome_after_render(outcome: str, format_status: Mapping[str, str], requested: set[str]) -> str:
1471
+ """A failed renderer must downgrade the outcome (never COMPLETE with a red format)."""
1472
+ actionable = [fmt for fmt in requested if format_status.get(fmt) not in {"not_applicable", "blocked"}]
1473
+ blocked = [fmt for fmt in requested if format_status.get(fmt) == "blocked"]
1474
+ if not actionable:
1475
+ return OUTCOME_BLOCKED if blocked and outcome == OUTCOME_COMPLETE else outcome
1476
+ complete = [fmt for fmt in actionable if format_status.get(fmt) == "complete"]
1477
+ failed = [fmt for fmt in actionable if format_status.get(fmt) == "failed"]
1478
+ if blocked and not failed and outcome == OUTCOME_COMPLETE:
1479
+ return OUTCOME_PARTIAL if complete else OUTCOME_BLOCKED
1480
+ if not failed:
1481
+ return outcome
1482
+ if outcome in {OUTCOME_BLOCKED, OUTCOME_FAILED}:
1483
+ return outcome
1484
+ if complete:
1485
+ return OUTCOME_PARTIAL
1486
+ return OUTCOME_FAILED
1487
+
1488
+ def _cloud_text_allowed(self, bundle: ExtractionBundleV2) -> bool:
1489
+ """Whether this run may use cloud text generation (opt-in, per platform)."""
1490
+ try:
1491
+ platform = bundle.sources[0].platform if bundle.sources else "youtube"
1492
+ return bool(self.settings.cloud_allowed(platform, "text"))
1493
+ except Exception: # pragma: no cover - defensive
1494
+ return False
1495
+
1496
+ # -------------------------------------------------------------- persist
1497
+ def _write_manifest(
1498
+ self,
1499
+ bundle: ExtractionBundleV2,
1500
+ output_dir: Path,
1501
+ *,
1502
+ requested: set[str],
1503
+ written: Mapping[str, Path],
1504
+ outcome: Outcome,
1505
+ format_status: Mapping[str, str],
1506
+ format_reasons: Mapping[str, str],
1507
+ reasons: Sequence[str],
1508
+ refresh: bool,
1509
+ previous_manifest: Any | None = None,
1510
+ doc_language: str | None = None,
1511
+ acquisition_mode: str = "platform",
1512
+ ) -> Path | None:
1513
+ completeness = bundle.contents[0].completeness.status if bundle.contents else "unknown"
1514
+ acquire_stage = {"complete": "complete", "partial": "partial"}.get(completeness, "unknown")
1515
+ renderable = [fmt for fmt in requested if format_status.get(fmt) != "not_applicable"]
1516
+ render_ok = [fmt for fmt in renderable if format_status.get(fmt) == "complete"]
1517
+ if not renderable:
1518
+ render_stage = "complete"
1519
+ elif len(render_ok) == len(renderable):
1520
+ render_stage = "complete"
1521
+ elif render_ok:
1522
+ render_stage = "partial"
1523
+ else:
1524
+ render_stage = "failed"
1525
+ source_revision = bundle.contents[0].source_revision if bundle.contents else None
1526
+ manifest: dict[str, Any] = {
1527
+ "schema_version": 2,
1528
+ "source_ids": [source.source_id for source in bundle.sources],
1529
+ "source_revision": source_revision,
1530
+ "request_fingerprint": _request_fingerprint(bundle, requested),
1531
+ "stages": {
1532
+ "resolve": "complete",
1533
+ "acquire": acquire_stage,
1534
+ "segment": "complete",
1535
+ "extract": "complete",
1536
+ "render": render_stage,
1537
+ },
1538
+ "outputs": {
1539
+ fmt: _output_entry(
1540
+ fmt,
1541
+ status=format_status.get(fmt, "pending"),
1542
+ path=written.get(fmt),
1543
+ output_dir=output_dir,
1544
+ source_revision=source_revision,
1545
+ reason=format_reasons.get(fmt),
1546
+ )
1547
+ for fmt in sorted(requested)
1548
+ },
1549
+ "outcome": outcome,
1550
+ "reasons": list(reasons),
1551
+ "refresh": bool(refresh),
1552
+ "remote_tasks": dict(getattr(previous_manifest, "remote_tasks", {}) or {}),
1553
+ "refresh_epoch": int(getattr(previous_manifest, "refresh_epoch", 0) or 0) + (1 if refresh else 0),
1554
+ "updated_at": _utc_now(),
1555
+ }
1556
+ from ytk.pipeline.state import (
1557
+ ArtifactStoreError,
1558
+ ManifestV2,
1559
+ mark_stage,
1560
+ new_manifest,
1561
+ record_output,
1562
+ )
1563
+
1564
+ manifest_model = new_manifest(
1565
+ source_ids=[source.source_id for source in bundle.sources],
1566
+ request_fingerprint=manifest["request_fingerprint"],
1567
+ source_revision=manifest.get("source_revision"),
1568
+ refresh_epoch=manifest.get("refresh_epoch", 0),
1569
+ )
1570
+ for stage_name, stage_status in manifest.get("stages", {}).items():
1571
+ manifest_model = mark_stage(manifest_model, stage_name, stage_status)
1572
+ entries = {
1573
+ fmt: _output_entry(
1574
+ fmt,
1575
+ status=format_status.get(fmt, "pending"),
1576
+ path=written.get(fmt),
1577
+ output_dir=output_dir,
1578
+ source_revision=source_revision,
1579
+ reason=format_reasons.get(fmt),
1580
+ )
1581
+ for fmt in sorted(requested)
1582
+ }
1583
+ for fmt, entry in entries.items():
1584
+ manifest_model = record_output(
1585
+ manifest_model,
1586
+ fmt,
1587
+ status=entry["status"],
1588
+ path=entry["path"],
1589
+ sha256=entry["sha256"],
1590
+ validator_version=entry["validator_version"],
1591
+ checker_version=entry.get("checker_version"),
1592
+ reason=entry.get("reason"),
1593
+ reason_code=entry.get("reason_code"),
1594
+ evidence_fingerprint=entry.get("evidence_fingerprint"),
1595
+ )
1596
+ for fmt, entry_model in manifest_model.outputs.items():
1597
+ extra = entries.get(fmt, {})
1598
+ for key in ("kind", "file_count", "tree_sha256"):
1599
+ if key in extra:
1600
+ if entry_model.__pydantic_extra__ is None:
1601
+ entry_model.__pydantic_extra__ = {}
1602
+ entry_model.__pydantic_extra__[key] = extra[key]
1603
+ payload = manifest_model.model_dump()
1604
+ current_manifest = self._read_previous_manifest(output_dir)
1605
+ if outcome not in {OUTCOME_COMPLETE, OUTCOME_NO_KNOWLEDGE}:
1606
+ checkpoint = getattr(current_manifest, "acquisition_checkpoint", None)
1607
+ if isinstance(checkpoint, dict):
1608
+ payload["acquisition_checkpoint"] = checkpoint
1609
+ payload.update(
1610
+ {
1611
+ "outcome": outcome,
1612
+ "reasons": list(reasons),
1613
+ "refresh": bool(refresh),
1614
+ "remote_tasks": dict(getattr(current_manifest, "remote_tasks", {}) or manifest.get("remote_tasks", {})),
1615
+ "render_fingerprints": {
1616
+ fmt: render_request_fingerprint(fmt, self.settings, doc_language, bundle.sources[0].platform)
1617
+ for fmt in requested
1618
+ },
1619
+ "content_cache": {
1620
+ "origin": acquisition_mode,
1621
+ "processing_fingerprint": self._processing_fingerprint(),
1622
+ **{
1623
+ filename: _sha256_file(output_dir / filename)
1624
+ for filename in ("knowledge.json", "content.json", "source.json")
1625
+ },
1626
+ },
1627
+ "updated_at": _utc_now(),
1628
+ }
1629
+ )
1630
+ manifest_model = ManifestV2.model_validate(payload)
1631
+ store = self._store(output_dir)
1632
+ if store is not None:
1633
+ try:
1634
+ # write_manifest validates every claimed-complete output, asserts
1635
+ # ownership and writes LAST — the fail-closed layer stays active.
1636
+ store.write_manifest(manifest_model)
1637
+ return output_dir / "manifest.json"
1638
+ except ArtifactStoreError:
1639
+ raise
1640
+ except Exception as exc: # never lose the run over manifest plumbing
1641
+ LOGGER.warning("artifact store manifest write failed: %s", exc)
1642
+ path = output_dir / "manifest.json"
1643
+ _atomic_write_json(path, payload)
1644
+ return path
1645
+
1646
+ # -------------------------------------------------------------- helpers
1647
+ @staticmethod
1648
+ def _load_fixture_content(path: Path) -> FixtureContent:
1649
+ if path.is_dir():
1650
+ candidate = path / "content.json"
1651
+ if not candidate.is_file():
1652
+ raise UserFacingError(f"fixture directory {path} has no content.json")
1653
+ path = candidate
1654
+ payload = json.loads(path.read_text(encoding="utf-8"))
1655
+ return FixtureContent.model_validate(payload)
1656
+
1657
+
1658
+ def _split_runner_output(output: Any) -> tuple[list[ContentBlock], dict[str, object]]:
1659
+ """Accept either ``list[ContentBlock]`` or ``(blocks, metadata)`` from a runner."""
1660
+ if isinstance(output, tuple) and len(output) == 2:
1661
+ blocks, metadata = output
1662
+ return list(blocks or []), dict(metadata or {})
1663
+ return list(output or []), {}
1664
+
1665
+
1666
+ def _no_knowledge_gate(bundle: ExtractionBundleV2, completeness: Any) -> bool:
1667
+ """NO_KNOWLEDGE requires the formal gate when the content model allows it."""
1668
+ try:
1669
+ from ytk.quality.completeness import decide_no_knowledge
1670
+
1671
+ if not bundle.contents:
1672
+ return completeness is not None and completeness.status == "complete"
1673
+ return decide_no_knowledge(bundle.contents[0], bundle.knowledge)
1674
+ except Exception: # pragma: no cover - defensive: fall back to strict check
1675
+ return (
1676
+ completeness is not None
1677
+ and completeness.status == "complete"
1678
+ and completeness.reason_code is None
1679
+ )
1680
+
1681
+
1682
+ def _default_filename(fmt: str) -> str:
1683
+ return {
1684
+ "json": "knowledge.json",
1685
+ "jsonl": "knowledge.jsonl",
1686
+ "md": "notes.md",
1687
+ "markdown": "notes.md",
1688
+ "obsidian": "obsidian.md",
1689
+ "summary": "summary.md",
1690
+ "doc": "knowledge.md",
1691
+ "skill": "skill",
1692
+ }.get(fmt, f"{fmt}.out")
1693
+
1694
+
1695
+ #: Bumped whenever a per-format artifact schema/validator changes. Resume must
1696
+ #: re-validate artifacts written by an older validator version (plan §4.5).
1697
+ FORMAT_VALIDATOR_VERSION = "v2-1"
1698
+
1699
+
1700
+ def _output_entry(
1701
+ fmt: str,
1702
+ *,
1703
+ status: str,
1704
+ path: Path | None,
1705
+ output_dir: Path,
1706
+ source_revision: str | None,
1707
+ reason: str | None = None,
1708
+ ) -> dict[str, Any]:
1709
+ is_directory = path is not None and path.is_dir()
1710
+ entry: dict[str, object] = {
1711
+ "status": status,
1712
+ "path": str(path.relative_to(output_dir)) if path is not None else None,
1713
+ # A skill package is a directory artifact; it has no single-file digest,
1714
+ # so it records its kind and a deterministic file-count instead.
1715
+ "kind": "directory" if is_directory else ("file" if path is not None else None),
1716
+ "sha256": _sha256_file(path) if path is not None and not is_directory else None,
1717
+ "validator_version": FORMAT_VALIDATOR_VERSION,
1718
+ }
1719
+ if is_directory and path is not None:
1720
+ from ytk.pipeline.state import directory_digest
1721
+
1722
+ entry["file_count"] = sum(1 for item in path.rglob("*") if item.is_file())
1723
+ entry["tree_sha256"] = directory_digest(path)
1724
+ if status == "not_applicable":
1725
+ # An inapplicable format still records WHY and against which evidence and
1726
+ # checker version, so a later resume re-judges only when those change.
1727
+ entry["reason"] = reason or "OUTPUTS_NOT_APPLICABLE"
1728
+ entry["reason_code"] = reason or "OUTPUTS_NOT_APPLICABLE"
1729
+ entry["evidence_fingerprint"] = source_revision
1730
+ entry["checker_version"] = FORMAT_VALIDATOR_VERSION
1731
+ elif reason:
1732
+ entry["reason"] = reason
1733
+ entry["reason_code"] = reason
1734
+ return entry
1735
+
1736
+
1737
+ def render_request_fingerprint(
1738
+ fmt: str, settings: AppSettings, doc_language: str | None, platform: str
1739
+ ) -> str:
1740
+ """Rendering options invalidate outputs without invalidating recognition."""
1741
+ from ytk.renderers.knowledge_doc_renderer import detect_doc_language
1742
+
1743
+ payload: dict[str, Any] = {
1744
+ "format": fmt, "validator": FORMAT_VALIDATOR_VERSION,
1745
+ "language": detect_doc_language(doc_language or settings.doc_language),
1746
+ "extraction_implementation": extraction_implementation_fingerprint(),
1747
+ }
1748
+ if fmt == "doc":
1749
+ payload.update({
1750
+ "model": settings.minimax_model, "region": settings.minimax_region,
1751
+ "max_tokens": settings.knowledge_doc_max_tokens,
1752
+ "cloud_allowed": settings.cloud_allowed(cast(Any, platform), "text"),
1753
+ "renderer": _sha256_file(Path(__file__).parents[1] / "renderers" / "knowledge_doc_renderer.py"),
1754
+ })
1755
+ return "sha256:" + hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
1756
+
1757
+
1758
+ def _request_fingerprint(bundle: ExtractionBundleV2, requested: set[str]) -> str:
1759
+ payload = json.dumps(
1760
+ {
1761
+ "formats": sorted(requested),
1762
+ "revision": [content.source_revision for content in bundle.contents],
1763
+ "blocks": [len(content.blocks) for content in bundle.contents],
1764
+ },
1765
+ sort_keys=True,
1766
+ )
1767
+ return "sha256:" + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]
1768
+
1769
+
1770
+ def _utc_now() -> str:
1771
+ from datetime import datetime
1772
+
1773
+ return datetime.now(UTC).isoformat(timespec="seconds")
1774
+
1775
+
1776
+ def _atomic_write_json(path: Path, payload: object) -> None:
1777
+ fd, tmp_name = tempfile.mkstemp(prefix=path.name + ".", suffix=".tmp", dir=path.parent)
1778
+ try:
1779
+ with os.fdopen(fd, "w", encoding="utf-8") as handle:
1780
+ handle.write(json.dumps(payload, indent=2, ensure_ascii=False) + "\n")
1781
+ os.replace(tmp_name, path)
1782
+ except BaseException:
1783
+ try:
1784
+ os.unlink(tmp_name)
1785
+ except OSError:
1786
+ pass
1787
+ raise
1788
+
1789
+
1790
+ def _copy_current_asset(source: Path, target: Path) -> None:
1791
+ """Replace stale media atomically while retaining identical cached material."""
1792
+ if target.is_symlink() or target.parent.is_symlink() or not target.parent.resolve().is_relative_to(target.parent.parent.resolve()):
1793
+ raise UserFacingError("media destination uses a symbolic link or points outside the task package")
1794
+ if target.is_file() and _sha256_file(source) == _sha256_file(target):
1795
+ return
1796
+ fd, temporary = tempfile.mkstemp(prefix=target.name + ".", suffix=".tmp", dir=target.parent)
1797
+ os.close(fd)
1798
+ try:
1799
+ shutil.copyfile(source, temporary)
1800
+ os.replace(temporary, target)
1801
+ finally:
1802
+ Path(temporary).unlink(missing_ok=True)
1803
+
1804
+
1805
+ def build_v2_pipeline(
1806
+ settings: AppSettings | None = None,
1807
+ *,
1808
+ registry: PlatformRegistry | None = None,
1809
+ access: AccessContext | None = None,
1810
+ renderers: Mapping[str, Callable[..., None]] | None = None,
1811
+ coarse_atoms: bool = False,
1812
+ ) -> MultiPlatformPipeline:
1813
+ """Factory used by the CLI; wires the built-in adapters when available."""
1814
+ settings = settings or AppSettings()
1815
+ adapters = _builtin_adapters(settings)
1816
+ renderer_map = dict(renderers or {})
1817
+ if not renderer_map:
1818
+ try:
1819
+ from ytk.renderers.v2 import build_v2_renderers
1820
+
1821
+ renderer_map = build_v2_renderers()
1822
+ except Exception as exc: # renderers not built yet / optional dep missing
1823
+ LOGGER.debug("v2 renderers unavailable: %s", exc)
1824
+ from ytk.pipeline.runners import make_artifact_store_factory, make_asr_runner, make_ocr_runner
1825
+
1826
+ return MultiPlatformPipeline(
1827
+ settings=settings,
1828
+ registry=registry or PlatformRegistry.default(adapters=adapters),
1829
+ access=access,
1830
+ renderers=renderer_map,
1831
+ coarse_atoms=coarse_atoms,
1832
+ asr_runner=make_asr_runner(settings),
1833
+ ocr_runner=make_ocr_runner(settings) if settings.ocr_enabled else None,
1834
+ artifact_store_factory=make_artifact_store_factory(),
1835
+ )
1836
+
1837
+
1838
+ def _builtin_adapters(settings: AppSettings | None) -> dict[str, Any]:
1839
+ """Instantiate the adapters that exist in this build (lazy, optional deps)."""
1840
+ adapters: dict[str, Any] = {}
1841
+ for module_name, class_name, platform in (
1842
+ ("ytk.ingestion.adapters.youtube", "YouTubeAdapter", "youtube"),
1843
+ ("ytk.ingestion.adapters.tiktok", "TikTokAdapter", "tiktok"),
1844
+ ("ytk.ingestion.adapters.bilibili", "BilibiliAdapter", "bilibili"),
1845
+ ("ytk.ingestion.adapters.douyin", "DouyinAdapter", "douyin"),
1846
+ ("ytk.ingestion.adapters.xiaohongshu", "XiaohongshuAdapter", "xiaohongshu"),
1847
+ ):
1848
+ try:
1849
+ module = __import__(module_name, fromlist=[class_name])
1850
+ adapter_class = getattr(module, class_name)
1851
+ adapters[platform] = adapter_class(settings=settings)
1852
+ except Exception as exc: # missing optional dep / not implemented yet
1853
+ LOGGER.debug("adapter %s unavailable: %s", platform, exc)
1854
+ return adapters
1855
+
1856
+
1857
+ __all__ = [
1858
+ "FixtureContent",
1859
+ "MultiPlatformPipeline",
1860
+ "Outcome",
1861
+ "V2RunResult",
1862
+ "build_v2_pipeline",
1863
+ ]