kd-engine 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kd_engine-1.0.0.dist-info/METADATA +19 -0
- kd_engine-1.0.0.dist-info/RECORD +118 -0
- kd_engine-1.0.0.dist-info/WHEEL +5 -0
- kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
- kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
- kd_engine-1.0.0.dist-info/top_level.txt +1 -0
- ytk/__init__.py +2 -0
- ytk/__main__.py +21 -0
- ytk/batch/__init__.py +39 -0
- ytk/batch/runner.py +1149 -0
- ytk/cache/__init__.py +34 -0
- ytk/cache/v2.py +562 -0
- ytk/cleaning/density.py +40 -0
- ytk/cleaning/fillers.py +18 -0
- ytk/cleaning/sentence_boundary.py +30 -0
- ytk/cleaning/service.py +62 -0
- ytk/cli.py +1281 -0
- ytk/commands/__init__.py +3 -0
- ytk/commands/merge.py +600 -0
- ytk/config.py +757 -0
- ytk/dedup/service.py +166 -0
- ytk/doctor.py +891 -0
- ytk/dotenv.py +334 -0
- ytk/errors.py +38 -0
- ytk/extraction/service.py +451 -0
- ytk/extraction/strategies/base.py +16 -0
- ytk/extraction/strategies/code_block.py +285 -0
- ytk/extraction/strategies/heuristic.py +791 -0
- ytk/extraction/strategies/local_model.py +56 -0
- ytk/extraction/text_context.py +177 -0
- ytk/extraction/visual_context.py +159 -0
- ytk/ingestion/adapters/__init__.py +1 -0
- ytk/ingestion/adapters/_common.py +865 -0
- ytk/ingestion/adapters/base.py +160 -0
- ytk/ingestion/adapters/bilibili.py +271 -0
- ytk/ingestion/adapters/douyin.py +445 -0
- ytk/ingestion/adapters/tiktok.py +102 -0
- ytk/ingestion/adapters/xiaohongshu.py +758 -0
- ytk/ingestion/adapters/youtube.py +74 -0
- ytk/ingestion/browser_acquired.py +220 -0
- ytk/ingestion/http_client.py +43 -0
- ytk/ingestion/local_import.py +202 -0
- ytk/ingestion/media.py +468 -0
- ytk/ingestion/providers/asr.py +902 -0
- ytk/ingestion/providers/base.py +13 -0
- ytk/ingestion/providers/dtk_client.py +816 -0
- ytk/ingestion/providers/minimax.py +350 -0
- ytk/ingestion/providers/transcript_api.py +336 -0
- ytk/ingestion/providers/xhs_client.py +646 -0
- ytk/ingestion/providers/ytdlp.py +400 -0
- ytk/ingestion/registry.py +189 -0
- ytk/ingestion/resolver.py +417 -0
- ytk/ingestion/search.py +56 -0
- ytk/ingestion/service.py +167 -0
- ytk/ingestion/subtitles.py +602 -0
- ytk/ingestion/transcript_cache.py +212 -0
- ytk/ingestion/url_parser.py +53 -0
- ytk/ingestion/video_frames.py +292 -0
- ytk/ingestion/ytdlp_acquire.py +754 -0
- ytk/logging.py +383 -0
- ytk/models/common.py +20 -0
- ytk/models/content.py +451 -0
- ytk/models/knowledge.py +43 -0
- ytk/models/migrations.py +262 -0
- ytk/models/skill.py +13 -0
- ytk/models/source.py +265 -0
- ytk/models/topic.py +18 -0
- ytk/models/transcript.py +30 -0
- ytk/ocr/__init__.py +65 -0
- ytk/ocr/engine.py +245 -0
- ytk/ocr/layout.py +347 -0
- ytk/ocr/pipeline.py +440 -0
- ytk/ocr/rapidocr_engine.py +144 -0
- ytk/pipeline/acquisition_checkpoint.py +240 -0
- ytk/pipeline/artifacts.py +18 -0
- ytk/pipeline/orchestrator.py +598 -0
- ytk/pipeline/processing_identity.py +18 -0
- ytk/pipeline/retry.py +728 -0
- ytk/pipeline/runners.py +578 -0
- ytk/pipeline/state.py +1107 -0
- ytk/pipeline/v2.py +1863 -0
- ytk/py.typed +1 -0
- ytk/quality/__init__.py +9 -0
- ytk/quality/completeness.py +260 -0
- ytk/renderers/base.py +9 -0
- ytk/renderers/citation_alignment.py +617 -0
- ytk/renderers/evidence_formatter.py +362 -0
- ytk/renderers/json_renderer.py +13 -0
- ytk/renderers/jsonl_renderer.py +18 -0
- ytk/renderers/knowledge_doc_renderer.py +1984 -0
- ytk/renderers/markdown_renderer.py +540 -0
- ytk/renderers/obsidian_renderer.py +138 -0
- ytk/renderers/skill_package.py +21 -0
- ytk/renderers/source_links.py +309 -0
- ytk/renderers/summary_renderer.py +163 -0
- ytk/renderers/v2.py +171 -0
- ytk/renderers/v2_jsonl.py +165 -0
- ytk/renderers/v2_markdown.py +477 -0
- ytk/renderers/v2_obsidian.py +113 -0
- ytk/renderers/v2_summary.py +220 -0
- ytk/segmentation/change_points.py +50 -0
- ytk/segmentation/embeddings.py +45 -0
- ytk/segmentation/service.py +318 -0
- ytk/skill_builder/__init__.py +1 -0
- ytk/skill_builder/builder.py +419 -0
- ytk/skill_builder/templates/SKILL.md.j2 +85 -0
- ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
- ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
- ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
- ytk/skill_builder/templates/references/commands.md.j2 +34 -0
- ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
- ytk/skill_builder/templates/references/configs.md.j2 +31 -0
- ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
- ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
- ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
- ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
- ytk/skill_builder/templates.py +29 -0
- ytk/text_normalization.py +44 -0
ytk/pipeline/v2.py
ADDED
|
@@ -0,0 +1,1863 @@
|
|
|
1
|
+
"""v2.0 multi-platform pipeline (T11.2 / T12.2 / T33.x integration).
|
|
2
|
+
|
|
3
|
+
Flow: resolve -> acquire (platform adapter) -> content bundle -> clean ->
|
|
4
|
+
segment -> extract -> render -> atomic manifest.
|
|
5
|
+
|
|
6
|
+
Two acquisition paths share everything after "content bundle":
|
|
7
|
+
|
|
8
|
+
* **live**: a :class:`~ytk.ingestion.adapters.base.PlatformAdapter` acquires the
|
|
9
|
+
source (yt-dlp / DTK / xiaohongshu-mcp / browser-derived fixture);
|
|
10
|
+
* **fixture-content**: a JSON descriptor (see :class:`FixtureContent`) describes
|
|
11
|
+
already-acquired content (real or recorded), including local media paths. This
|
|
12
|
+
is the offline/end-to-end path mandated by the plan (``--fixture-content``) and
|
|
13
|
+
the path used for acceptance evidence captured through the real browser.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import hashlib
|
|
19
|
+
import json
|
|
20
|
+
import logging
|
|
21
|
+
import os
|
|
22
|
+
import re as _re
|
|
23
|
+
import shutil
|
|
24
|
+
import tempfile
|
|
25
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
26
|
+
from contextlib import nullcontext
|
|
27
|
+
from dataclasses import dataclass, field
|
|
28
|
+
from datetime import UTC
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import TYPE_CHECKING, Any, Final, Literal, cast
|
|
31
|
+
|
|
32
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
33
|
+
|
|
34
|
+
from ytk.cleaning.service import CleaningService
|
|
35
|
+
from ytk.config import AppSettings
|
|
36
|
+
from ytk.errors import (
|
|
37
|
+
AnalysisBlockedError,
|
|
38
|
+
ModelUnavailableError,
|
|
39
|
+
ServiceNotConfiguredError,
|
|
40
|
+
UserFacingError,
|
|
41
|
+
)
|
|
42
|
+
from ytk.extraction.service import BlockExtraction, ExtractionService
|
|
43
|
+
from ytk.ingestion.registry import PlatformRegistry
|
|
44
|
+
from ytk.ingestion.resolver import (
|
|
45
|
+
access_tokens_from_url,
|
|
46
|
+
detect_platform,
|
|
47
|
+
extract_urls,
|
|
48
|
+
resolve_source,
|
|
49
|
+
)
|
|
50
|
+
from ytk.logging import redact_payload
|
|
51
|
+
from ytk.models.content import (
|
|
52
|
+
ContentBlock,
|
|
53
|
+
ContentBundle,
|
|
54
|
+
ContentCompleteness,
|
|
55
|
+
EvidenceRefV2,
|
|
56
|
+
ExtractionBundleV2,
|
|
57
|
+
KnowledgeAtomV2,
|
|
58
|
+
MediaAsset,
|
|
59
|
+
ProviderTrace,
|
|
60
|
+
dump_bundle_json,
|
|
61
|
+
make_block_id,
|
|
62
|
+
)
|
|
63
|
+
from ytk.models.source import AccessContext, ContentRef, ContentType, compute_source_revision
|
|
64
|
+
from ytk.pipeline.processing_identity import extraction_implementation_fingerprint
|
|
65
|
+
from ytk.segmentation.service import TopicSegmentationService
|
|
66
|
+
|
|
67
|
+
if TYPE_CHECKING: # pragma: no cover - typing only (avoids the registry import cycle)
|
|
68
|
+
from ytk.ingestion.adapters.base import PlatformAdapter
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
#: Metadata keys whose very name marks the value as credential-like. Value-shape
|
|
72
|
+
#: redaction (URLs, token patterns) alone is not enough: a bare ``SESSDATA`` or
|
|
73
|
+
#: ``signature`` string value survives ``redact_payload`` (adversarial review P0).
|
|
74
|
+
_SENSITIVE_META_KEY = _re.compile(
|
|
75
|
+
r"(xsec|mstoken|a_bogus|w_rid|bearer|api[-_]?key|token|cookie|session|sess|csrf|jct|"
|
|
76
|
+
r"secret|signature|authorization|password|credential)",
|
|
77
|
+
_re.IGNORECASE,
|
|
78
|
+
)
|
|
79
|
+
_REDACTED = "***REDACTED***"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _sanitize_metadata(value: object) -> object:
|
|
83
|
+
"""Redact credentials by key name AND by value shape, recursively."""
|
|
84
|
+
if isinstance(value, dict):
|
|
85
|
+
return {
|
|
86
|
+
key: _REDACTED if _SENSITIVE_META_KEY.search(str(key)) else _sanitize_metadata(item)
|
|
87
|
+
for key, item in value.items()
|
|
88
|
+
}
|
|
89
|
+
if isinstance(value, (list, tuple)):
|
|
90
|
+
return [_sanitize_metadata(item) for item in value]
|
|
91
|
+
if isinstance(value, str):
|
|
92
|
+
redacted = redact_payload(value)
|
|
93
|
+
return redacted if isinstance(redacted, str) else str(redacted)
|
|
94
|
+
return value
|
|
95
|
+
|
|
96
|
+
LOGGER = logging.getLogger(__name__)
|
|
97
|
+
|
|
98
|
+
OUTCOME_COMPLETE: Final = "COMPLETE"
|
|
99
|
+
OUTCOME_PARTIAL: Final = "PARTIAL"
|
|
100
|
+
OUTCOME_BLOCKED: Final = "BLOCKED"
|
|
101
|
+
OUTCOME_FAILED: Final = "FAILED"
|
|
102
|
+
OUTCOME_NO_KNOWLEDGE: Final = "NO_KNOWLEDGE"
|
|
103
|
+
|
|
104
|
+
Outcome = Literal["COMPLETE", "PARTIAL", "BLOCKED", "FAILED", "NO_KNOWLEDGE"]
|
|
105
|
+
|
|
106
|
+
#: formats that need real knowledge (as opposed to "not applicable" outcomes)
|
|
107
|
+
KNOWLEDGE_FORMATS = frozenset({"json", "md", "md_notes", "obsidian", "summary", "jsonl", "doc"})
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class FixtureContent(BaseModel):
|
|
111
|
+
"""``--fixture-content`` descriptor: already-acquired content, no network."""
|
|
112
|
+
|
|
113
|
+
model_config = ConfigDict(extra="forbid")
|
|
114
|
+
|
|
115
|
+
schema_version: Literal[2] = 2
|
|
116
|
+
source: ContentRef
|
|
117
|
+
metadata: dict[str, object] = Field(default_factory=dict)
|
|
118
|
+
blocks: list[ContentBlock] = Field(default_factory=list)
|
|
119
|
+
media: list[dict[str, object]] = Field(default_factory=list)
|
|
120
|
+
subtitle_text: str | None = None
|
|
121
|
+
completeness: ContentCompleteness = Field(
|
|
122
|
+
default_factory=lambda: ContentCompleteness(status="complete", checked=["fixture_content"])
|
|
123
|
+
)
|
|
124
|
+
provider_trace: list[ProviderTrace] = Field(default_factory=list)
|
|
125
|
+
chapters: list[dict[str, object]] = Field(default_factory=list)
|
|
126
|
+
|
|
127
|
+
@property
|
|
128
|
+
def content_type(self) -> ContentType:
|
|
129
|
+
return self.source.content_type
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class _DeterministicEmbeddingClient:
|
|
133
|
+
model_name = "deterministic-local"
|
|
134
|
+
|
|
135
|
+
def __init__(self, dimensions: int = 12) -> None:
|
|
136
|
+
self._dimensions = dimensions
|
|
137
|
+
|
|
138
|
+
def encode(self, texts: Sequence[str]) -> list[list[float]]:
|
|
139
|
+
import re
|
|
140
|
+
|
|
141
|
+
vectors = []
|
|
142
|
+
for text in texts:
|
|
143
|
+
vector = [0.0] * self._dimensions
|
|
144
|
+
for word in re.findall(r"[a-z0-9]+", text.casefold()):
|
|
145
|
+
vector[sum(ord(char) for char in word) % self._dimensions] += 1.0
|
|
146
|
+
vectors.append(vector)
|
|
147
|
+
return vectors
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@dataclass
|
|
151
|
+
class V2RunResult:
|
|
152
|
+
"""What one v2 run produced (kept out of the JSON models for flexibility)."""
|
|
153
|
+
|
|
154
|
+
bundle: ExtractionBundleV2
|
|
155
|
+
output_dir: Path
|
|
156
|
+
formats: set[str]
|
|
157
|
+
outcome: Outcome
|
|
158
|
+
format_status: dict[str, str] = field(default_factory=dict)
|
|
159
|
+
reasons: list[str] = field(default_factory=list)
|
|
160
|
+
manifest_path: Path | None = None
|
|
161
|
+
failed_formats: dict[str, str] = field(default_factory=dict)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _sha256_bytes(data: bytes) -> str:
|
|
165
|
+
return hashlib.sha256(data).hexdigest()
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _sha256_file(path: Path) -> str:
|
|
169
|
+
digest = hashlib.sha256()
|
|
170
|
+
with path.open("rb") as handle:
|
|
171
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
172
|
+
digest.update(chunk)
|
|
173
|
+
return digest.hexdigest()
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _share_url(value: str) -> str:
|
|
177
|
+
try:
|
|
178
|
+
return extract_urls(value)[0]
|
|
179
|
+
except ValueError:
|
|
180
|
+
return value
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
class MultiPlatformPipeline:
|
|
184
|
+
"""Assemble and persist a v2 knowledge product for any registered platform."""
|
|
185
|
+
|
|
186
|
+
def __init__(
|
|
187
|
+
self,
|
|
188
|
+
*,
|
|
189
|
+
settings: AppSettings | None = None,
|
|
190
|
+
registry: PlatformRegistry | None = None,
|
|
191
|
+
access: AccessContext | None = None,
|
|
192
|
+
renderers: Mapping[str, Callable[..., None]] | None = None,
|
|
193
|
+
asr_runner: Callable[..., list[ContentBlock]] | None = None,
|
|
194
|
+
ocr_runner: Callable[..., list[ContentBlock]] | None = None,
|
|
195
|
+
artifact_store_factory: Callable[[Path], Any] | None = None,
|
|
196
|
+
embedding_client: Any | None = None,
|
|
197
|
+
coarse_atoms: bool = False,
|
|
198
|
+
) -> None:
|
|
199
|
+
self.settings = settings or AppSettings()
|
|
200
|
+
self.registry = registry or PlatformRegistry.default()
|
|
201
|
+
self.access = access or AccessContext()
|
|
202
|
+
self.renderers = dict(renderers or {})
|
|
203
|
+
self.asr_runner = asr_runner
|
|
204
|
+
self.ocr_runner = ocr_runner
|
|
205
|
+
if artifact_store_factory is None:
|
|
206
|
+
from ytk.pipeline.runners import make_artifact_store_factory
|
|
207
|
+
|
|
208
|
+
artifact_store_factory = make_artifact_store_factory()
|
|
209
|
+
self.artifact_store_factory = artifact_store_factory
|
|
210
|
+
self.cleaning = CleaningService()
|
|
211
|
+
self.segmentation = TopicSegmentationService(
|
|
212
|
+
embedding_client=embedding_client or _DeterministicEmbeddingClient()
|
|
213
|
+
)
|
|
214
|
+
self.extraction = ExtractionService()
|
|
215
|
+
# 粗粒度分段:默认关闭。开启后,只有在完全抽不出原子时才生效。
|
|
216
|
+
self.coarse_atoms = bool(coarse_atoms)
|
|
217
|
+
|
|
218
|
+
# ------------------------------------------------------------------ run
|
|
219
|
+
def run(
|
|
220
|
+
self,
|
|
221
|
+
url: str,
|
|
222
|
+
*,
|
|
223
|
+
output_dir: Path,
|
|
224
|
+
formats: Sequence[str] | None = None,
|
|
225
|
+
fixture_content: Path | None = None,
|
|
226
|
+
refresh: bool = False,
|
|
227
|
+
continue_on_error: bool = True,
|
|
228
|
+
formats_explicit: bool = False,
|
|
229
|
+
doc_language: str | None = None,
|
|
230
|
+
platform: str | None = None,
|
|
231
|
+
content_type: ContentType | None = None,
|
|
232
|
+
preserve_existing_formats: bool = False,
|
|
233
|
+
) -> V2RunResult:
|
|
234
|
+
output_dir = Path(output_dir)
|
|
235
|
+
output_dir.mkdir(parents=True, exist_ok=True)
|
|
236
|
+
store = self._store(output_dir)
|
|
237
|
+
lock = store.locked(store.lock_timeout) if store is not None else nullcontext()
|
|
238
|
+
with lock: # hold the directory lock for the WHOLE task (TOCTOU fix, A39)
|
|
239
|
+
previous_manifest = self._read_previous_manifest(output_dir)
|
|
240
|
+
return self._run_inner(
|
|
241
|
+
url,
|
|
242
|
+
output_dir=output_dir,
|
|
243
|
+
formats=formats,
|
|
244
|
+
fixture_content=fixture_content,
|
|
245
|
+
refresh=refresh,
|
|
246
|
+
continue_on_error=continue_on_error,
|
|
247
|
+
formats_explicit=formats_explicit,
|
|
248
|
+
doc_language=doc_language,
|
|
249
|
+
platform=platform,
|
|
250
|
+
content_type=content_type,
|
|
251
|
+
previous_manifest=previous_manifest,
|
|
252
|
+
preserve_existing_formats=preserve_existing_formats,
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
def _run_inner(
|
|
256
|
+
self,
|
|
257
|
+
url: str,
|
|
258
|
+
*,
|
|
259
|
+
output_dir: Path,
|
|
260
|
+
formats: Sequence[str] | None,
|
|
261
|
+
fixture_content: Path | None,
|
|
262
|
+
refresh: bool,
|
|
263
|
+
continue_on_error: bool,
|
|
264
|
+
formats_explicit: bool,
|
|
265
|
+
doc_language: str | None,
|
|
266
|
+
platform: str | None,
|
|
267
|
+
content_type: ContentType | None,
|
|
268
|
+
previous_manifest: Any | None,
|
|
269
|
+
preserve_existing_formats: bool = False,
|
|
270
|
+
) -> V2RunResult:
|
|
271
|
+
resolved_tokens: dict[str, str] = {}
|
|
272
|
+
|
|
273
|
+
def capture_resolved_url(resolved_url: str) -> None:
|
|
274
|
+
resolved_tokens.update(access_tokens_from_url(resolved_url))
|
|
275
|
+
|
|
276
|
+
task_access = self._task_access(url, resolved_url_callback=capture_resolved_url)
|
|
277
|
+
deadline = self._make_deadline()
|
|
278
|
+
|
|
279
|
+
acquisition: Any | None = None
|
|
280
|
+
reused_content = False
|
|
281
|
+
if fixture_content is not None:
|
|
282
|
+
fixture = self._load_fixture_content(Path(fixture_content))
|
|
283
|
+
ref = fixture.source
|
|
284
|
+
self._assert_output_ownership(output_dir, previous_manifest, ref)
|
|
285
|
+
self._assert_no_unresolved_remote(previous_manifest)
|
|
286
|
+
if fixture.metadata.get("acquisition_mode") == "browser_video_live":
|
|
287
|
+
content, acquisition = self._browser_content_from_fixture(fixture, output_dir, url)
|
|
288
|
+
else:
|
|
289
|
+
content = self._content_from_fixture(fixture, output_dir)
|
|
290
|
+
else:
|
|
291
|
+
ref = self._resolve(url, platform=platform, content_type=content_type, access=task_access)
|
|
292
|
+
task_access = task_access.model_copy(deep=True, update={
|
|
293
|
+
"tokens": {**task_access.tokens, **resolved_tokens}, "resolved_url_callback": None,
|
|
294
|
+
})
|
|
295
|
+
self._assert_output_ownership(output_dir, previous_manifest, ref, allow_unresolved_variant=True)
|
|
296
|
+
self._assert_no_unresolved_remote(previous_manifest)
|
|
297
|
+
cached = self._reusable_content(output_dir, previous_manifest, ref) if preserve_existing_formats and not refresh else None
|
|
298
|
+
if cached is not None:
|
|
299
|
+
content = cached
|
|
300
|
+
ref = content.source
|
|
301
|
+
reused_content = True
|
|
302
|
+
else:
|
|
303
|
+
content, acquisition = self._acquired_content(
|
|
304
|
+
ref, output_dir, previous_manifest, task_access, refresh=refresh,
|
|
305
|
+
content_type=content_type,
|
|
306
|
+
)
|
|
307
|
+
ref = content.source
|
|
308
|
+
|
|
309
|
+
requested = self._resolve_formats(ref, formats)
|
|
310
|
+
if formats_explicit and not requested:
|
|
311
|
+
raise UserFacingError("explicitly requested format set is empty")
|
|
312
|
+
|
|
313
|
+
# ASR / OCR enrich the content *before* the revision is frozen so that
|
|
314
|
+
# evidence ids and the revision stay consistent.
|
|
315
|
+
if not reused_content:
|
|
316
|
+
self._check_deadline(deadline, "ocr")
|
|
317
|
+
content = self._maybe_ocr(content, output_dir)
|
|
318
|
+
self._check_deadline(deadline, "frame-ocr")
|
|
319
|
+
content = self._maybe_frame_ocr(content, output_dir)
|
|
320
|
+
self._check_deadline(deadline, "asr")
|
|
321
|
+
content = self._maybe_asr(content, output_dir)
|
|
322
|
+
if content.metadata.get("acquisition_mode") == "browser_video_live":
|
|
323
|
+
from ytk.ingestion.browser_acquired import assess_browser_transcript
|
|
324
|
+
content = assess_browser_transcript(content)
|
|
325
|
+
# Re-derive completeness from the evidence (never trust a declared
|
|
326
|
+
# "complete"): truncated subtitles, failed ASR chunks, cover-only and
|
|
327
|
+
# unverified narration must downgrade the run (T22.2 / A29).
|
|
328
|
+
if acquisition is not None:
|
|
329
|
+
content = self._assess_completeness(acquisition, content)
|
|
330
|
+
elif content.metadata.get("acquisition_mode") == "user_import":
|
|
331
|
+
from ytk.ingestion.local_import import assess_import
|
|
332
|
+
content = assess_import(content)
|
|
333
|
+
content = content.model_copy(update={"source_revision": compute_source_revision(
|
|
334
|
+
content.source.source_id,
|
|
335
|
+
{
|
|
336
|
+
"blocks": [block.model_dump(mode="json", exclude={"cleaned_text"}) for block in sorted(content.blocks, key=lambda block: (block.order, block.block_id))],
|
|
337
|
+
"assets": [(asset.media_type, asset.order, asset.sha256) for asset in sorted(content.assets, key=lambda asset: (asset.order, asset.asset_id))],
|
|
338
|
+
},
|
|
339
|
+
)})
|
|
340
|
+
|
|
341
|
+
bundle = self._sanitize_bundle(self._assemble(ref, content))
|
|
342
|
+
outcome, format_status, format_reasons, reasons = self._decide_outcomes(bundle, requested)
|
|
343
|
+
|
|
344
|
+
self._check_deadline(deadline, "render")
|
|
345
|
+
written = self._render(bundle, requested, output_dir, format_status, doc_language=doc_language)
|
|
346
|
+
if format_status.get("doc") == "blocked":
|
|
347
|
+
try:
|
|
348
|
+
status = json.loads((output_dir / ".knowledge-doc-status.json").read_text(encoding="utf-8"))
|
|
349
|
+
format_reasons["doc"] = str(status.get("reason") or "DOCUMENT_UNVERIFIED")
|
|
350
|
+
except (OSError, ValueError, TypeError):
|
|
351
|
+
format_reasons["doc"] = "DOCUMENT_UNVERIFIED"
|
|
352
|
+
reasons.append("doc BLOCKED: " + format_reasons["doc"])
|
|
353
|
+
outcome = cast(Outcome, self._adjust_outcome_after_render(outcome, format_status, requested))
|
|
354
|
+
manifest_formats = set(requested)
|
|
355
|
+
if (
|
|
356
|
+
preserve_existing_formats
|
|
357
|
+
and previous_manifest is not None
|
|
358
|
+
and previous_manifest.source_revision == content.source_revision
|
|
359
|
+
and bundle.knowledge
|
|
360
|
+
):
|
|
361
|
+
store = self._store(output_dir)
|
|
362
|
+
if store is not None:
|
|
363
|
+
for fmt, entry in previous_manifest.outputs.items():
|
|
364
|
+
same_settings = (
|
|
365
|
+
fmt == "json" or getattr(previous_manifest, "render_fingerprints", {}).get(fmt)
|
|
366
|
+
== render_request_fingerprint(fmt, self.settings, doc_language, ref.platform)
|
|
367
|
+
)
|
|
368
|
+
if fmt not in requested and same_settings and store.output_valid(fmt, validator_version=FORMAT_VALIDATOR_VERSION):
|
|
369
|
+
manifest_formats.add(fmt)
|
|
370
|
+
format_status[fmt] = "complete"
|
|
371
|
+
written[fmt] = output_dir / entry.path
|
|
372
|
+
self._remove_stale_artifacts(previous_manifest, written, output_dir)
|
|
373
|
+
manifest_path = self._write_manifest(
|
|
374
|
+
bundle,
|
|
375
|
+
output_dir,
|
|
376
|
+
requested=manifest_formats,
|
|
377
|
+
written=written,
|
|
378
|
+
outcome=outcome,
|
|
379
|
+
format_status=format_status,
|
|
380
|
+
format_reasons=format_reasons,
|
|
381
|
+
reasons=reasons,
|
|
382
|
+
refresh=refresh,
|
|
383
|
+
previous_manifest=previous_manifest,
|
|
384
|
+
doc_language=doc_language,
|
|
385
|
+
acquisition_mode=("user-import" if content.metadata.get("acquisition_mode") == "user_import"
|
|
386
|
+
else ("browser-video-live" if content.metadata.get("acquisition_mode") == "browser_video_live"
|
|
387
|
+
else ("fixture-content" if fixture_content is not None else "platform"))),
|
|
388
|
+
)
|
|
389
|
+
return V2RunResult(
|
|
390
|
+
bundle=bundle,
|
|
391
|
+
output_dir=output_dir,
|
|
392
|
+
formats=requested,
|
|
393
|
+
outcome=outcome,
|
|
394
|
+
format_status=format_status,
|
|
395
|
+
reasons=reasons,
|
|
396
|
+
manifest_path=manifest_path,
|
|
397
|
+
)
|
|
398
|
+
|
|
399
|
+
# ------------------------------------------------------------ resolution
|
|
400
|
+
def _resolve(self, url: str, *, platform: str | None, content_type: ContentType | None,
|
|
401
|
+
access: AccessContext | None = None) -> ContentRef:
|
|
402
|
+
from ytk.ingestion.registry import _REGISTRATION_ORDER # local: avoid import cycle
|
|
403
|
+
|
|
404
|
+
detected: str | None = detect_platform(_share_url(url))
|
|
405
|
+
if detected is None:
|
|
406
|
+
try:
|
|
407
|
+
detected = resolve_source(url, content_type=content_type).platform
|
|
408
|
+
except UserFacingError:
|
|
409
|
+
detected = platform
|
|
410
|
+
platform = platform or detected
|
|
411
|
+
if platform is None:
|
|
412
|
+
raise UserFacingError(f"cannot determine the platform of {url!r}")
|
|
413
|
+
if platform not in _REGISTRATION_ORDER:
|
|
414
|
+
raise UserFacingError(f"unsupported platform {platform!r}")
|
|
415
|
+
adapter = self.registry.get(platform).adapter
|
|
416
|
+
if adapter is not None:
|
|
417
|
+
# PlatformRegistration.adapter is stored as ``object`` (pydantic
|
|
418
|
+
# cannot validate a Protocol); the registry only ever installs
|
|
419
|
+
# PlatformAdapter instances.
|
|
420
|
+
return cast("PlatformAdapter", adapter).resolve(url, access or self.access)
|
|
421
|
+
return resolve_source(url, platform=platform, content_type=content_type) # type: ignore[arg-type]
|
|
422
|
+
|
|
423
|
+
def _convert_sidecar_error(self, exc: Exception, output_dir: Path) -> Exception | None:
|
|
424
|
+
"""Convert sidecar typed errors into blocked outcomes WITH state recorded.
|
|
425
|
+
|
|
426
|
+
A DTK task timeout carries a resumable task id: recording it here is what
|
|
427
|
+
makes the unresolved-remote guard (A36) real on the live path — the next
|
|
428
|
+
run sees the record and refuses to re-dispatch instead of starting over.
|
|
429
|
+
"""
|
|
430
|
+
name = type(exc).__name__
|
|
431
|
+
if name not in {
|
|
432
|
+
"DtkTaskTimeoutError",
|
|
433
|
+
"DtkAuthRequiredError",
|
|
434
|
+
"DtkSubmissionUnknownError",
|
|
435
|
+
"DtkTaskFailedError",
|
|
436
|
+
"DtkServiceUnavailableError",
|
|
437
|
+
"DtkRateLimitedError",
|
|
438
|
+
"DtkContentUnavailableError",
|
|
439
|
+
"DtkApiChangedError",
|
|
440
|
+
"XhsUrlExpiredError",
|
|
441
|
+
}:
|
|
442
|
+
return None
|
|
443
|
+
if name == "DtkTaskTimeoutError":
|
|
444
|
+
store = self._store(output_dir)
|
|
445
|
+
if store is not None:
|
|
446
|
+
try:
|
|
447
|
+
store.record_remote_task(
|
|
448
|
+
"dtk",
|
|
449
|
+
task_id=getattr(exc, "task_id", None),
|
|
450
|
+
status="running",
|
|
451
|
+
deadline=getattr(exc, "deadline", None),
|
|
452
|
+
)
|
|
453
|
+
except Exception as record_exc: # pragma: no cover - never crash on state
|
|
454
|
+
LOGGER.warning("could not record remote task: %s", record_exc)
|
|
455
|
+
elif name in {"DtkAuthRequiredError", "DtkRateLimitedError", "DtkContentUnavailableError", "DtkTaskFailedError"}:
|
|
456
|
+
store = self._store(output_dir)
|
|
457
|
+
if store is not None:
|
|
458
|
+
saved = store.read_manifest()
|
|
459
|
+
task = saved.remote_tasks.get("dtk", {}) if saved is not None else {}
|
|
460
|
+
if not task.get("task_id") or name == "DtkTaskFailedError":
|
|
461
|
+
store.record_remote_task("dtk", status="failed", task_id=task.get("task_id"))
|
|
462
|
+
reason = getattr(exc, "reason_code", None) or name
|
|
463
|
+
return AnalysisBlockedError(f"{reason}: {exc}")
|
|
464
|
+
|
|
465
|
+
def _make_deadline(self) -> Any | None:
|
|
466
|
+
seconds = float(getattr(self.settings, "task_deadline_s", 0) or 0)
|
|
467
|
+
if seconds <= 0:
|
|
468
|
+
return None
|
|
469
|
+
from ytk.pipeline.retry import TaskDeadline
|
|
470
|
+
|
|
471
|
+
return TaskDeadline(seconds)
|
|
472
|
+
|
|
473
|
+
@staticmethod
|
|
474
|
+
def _check_deadline(deadline: Any | None, stage: str) -> None:
|
|
475
|
+
if deadline is None:
|
|
476
|
+
return
|
|
477
|
+
try:
|
|
478
|
+
deadline.check()
|
|
479
|
+
except Exception as exc:
|
|
480
|
+
from ytk.errors import AnalysisBlockedError
|
|
481
|
+
|
|
482
|
+
raise AnalysisBlockedError(f"TASK_DEADLINE_EXCEEDED at stage {stage}: {exc}") from exc
|
|
483
|
+
|
|
484
|
+
def _task_access(self, url: str, *, resolved_url_callback: Callable[[str], None] | None = None) -> AccessContext:
|
|
485
|
+
"""Build the per-task credential context from settings + the share URL.
|
|
486
|
+
|
|
487
|
+
Platform isolation matters here (plan §4.6): the legacy global YouTube
|
|
488
|
+
cookie/proxy must never leak to another platform, and an XHS ``xsec_token``
|
|
489
|
+
carried by the share URL belongs in the protected access context, never in
|
|
490
|
+
a public artifact.
|
|
491
|
+
"""
|
|
492
|
+
candidate = _share_url(url)
|
|
493
|
+
platform = detect_platform(candidate)
|
|
494
|
+
cookies_file = self.access.cookies_file
|
|
495
|
+
cookies_from_browser = self.access.cookies_from_browser
|
|
496
|
+
proxy = self.access.proxy
|
|
497
|
+
headers = dict(self.access.headers)
|
|
498
|
+
tokens = dict(self.access.tokens)
|
|
499
|
+
if platform is not None:
|
|
500
|
+
try:
|
|
501
|
+
cookies_file = self.settings.cookies_file_for(platform) or cookies_file
|
|
502
|
+
cookies_from_browser = (
|
|
503
|
+
self.settings.cookies_from_browser_for(platform) or cookies_from_browser
|
|
504
|
+
)
|
|
505
|
+
proxy = self.settings.proxy_for(platform) or proxy
|
|
506
|
+
except Exception: # pragma: no cover - defensive
|
|
507
|
+
pass
|
|
508
|
+
try:
|
|
509
|
+
tokens.update(access_tokens_from_url(candidate))
|
|
510
|
+
except Exception: # pragma: no cover - defensive
|
|
511
|
+
pass
|
|
512
|
+
return AccessContext(
|
|
513
|
+
cookies_file=cookies_file,
|
|
514
|
+
cookies_from_browser=cookies_from_browser,
|
|
515
|
+
proxy=proxy,
|
|
516
|
+
headers=headers,
|
|
517
|
+
tokens=tokens,
|
|
518
|
+
resolved_url_callback=resolved_url_callback,
|
|
519
|
+
)
|
|
520
|
+
|
|
521
|
+
def _acquire(
|
|
522
|
+
self, ref: ContentRef, work_dir: Path | None = None, access: AccessContext | None = None,
|
|
523
|
+
refresh: bool = False,
|
|
524
|
+
) -> Any:
|
|
525
|
+
from ytk.ingestion.adapters.base import AcquisitionContext
|
|
526
|
+
|
|
527
|
+
adapter = self.registry.require_adapter(ref.platform)
|
|
528
|
+
if work_dir is None:
|
|
529
|
+
work_dir = Path(".")
|
|
530
|
+
work_dir.mkdir(parents=True, exist_ok=True)
|
|
531
|
+
task_options: dict[str, Any] = {}
|
|
532
|
+
if ref.platform == "douyin":
|
|
533
|
+
from ytk.pipeline.state import new_manifest
|
|
534
|
+
|
|
535
|
+
store = self._store(work_dir.parent.parent)
|
|
536
|
+
if store is None:
|
|
537
|
+
raise UserFacingError("cannot safely persist the remote submission state")
|
|
538
|
+
saved = store.read_manifest()
|
|
539
|
+
if saved is None:
|
|
540
|
+
saved = new_manifest(source_ids=[ref.source_id], request_fingerprint="")
|
|
541
|
+
store.write_manifest(saved)
|
|
542
|
+
remote = saved.remote_tasks.get("dtk", {})
|
|
543
|
+
if remote.get("status") not in {"failed", "cancelled", "canceled"} and not (
|
|
544
|
+
refresh and remote.get("status") == "complete"
|
|
545
|
+
):
|
|
546
|
+
task_options["remote_task_id"] = remote.get("task_id")
|
|
547
|
+
|
|
548
|
+
def record_remote(status: str, task_id: str | None) -> None:
|
|
549
|
+
store.record_remote_task("dtk", status=status, task_id=task_id)
|
|
550
|
+
|
|
551
|
+
task_options["remote_task_callback"] = record_remote
|
|
552
|
+
context = AcquisitionContext(
|
|
553
|
+
access=access or self.access,
|
|
554
|
+
work_dir=str(work_dir),
|
|
555
|
+
languages=getattr(self.settings, "transcript_languages", ("en", "zh-Hans", "zh")),
|
|
556
|
+
asr_enabled=True,
|
|
557
|
+
ocr_enabled=bool(getattr(self.settings, "ocr_enabled", False)),
|
|
558
|
+
media_budget_mb=self.settings.max_media_mb,
|
|
559
|
+
frame_budget={"max_frames": self.settings.frame_budget},
|
|
560
|
+
**task_options,
|
|
561
|
+
)
|
|
562
|
+
return adapter.acquire(ref, context)
|
|
563
|
+
|
|
564
|
+
# --------------------------------------------------------------- content
|
|
565
|
+
def _processing_fingerprint(self) -> str:
|
|
566
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
567
|
+
|
|
568
|
+
from ytk.pipeline.runners import _asr_model_fingerprint, resolve_asr_model
|
|
569
|
+
from ytk.ingestion.providers.asr import asr_decoding_profile
|
|
570
|
+
|
|
571
|
+
model: object = None
|
|
572
|
+
try:
|
|
573
|
+
selected = resolve_asr_model(self.settings)
|
|
574
|
+
model = {"path": selected, "files": _asr_model_fingerprint(selected)}
|
|
575
|
+
except (ModelUnavailableError, OSError):
|
|
576
|
+
pass
|
|
577
|
+
ocr: object = None
|
|
578
|
+
if self.settings.ocr_enabled:
|
|
579
|
+
from ytk.ocr.engine import resolve_model_paths
|
|
580
|
+
|
|
581
|
+
try:
|
|
582
|
+
ocr = resolve_model_paths(self.settings.ocr_model_dir).fingerprint()
|
|
583
|
+
except (ModelUnavailableError, OSError):
|
|
584
|
+
pass
|
|
585
|
+
payload = {
|
|
586
|
+
"asr_backend": self.settings.asr_backend,
|
|
587
|
+
"asr_model": self.settings.asr_model,
|
|
588
|
+
"asr_decoding": asr_decoding_profile(self.settings),
|
|
589
|
+
"faster_whisper_model": self.settings.faster_whisper_model,
|
|
590
|
+
"whisper_cpp_model": self.settings.whisper_cpp_model,
|
|
591
|
+
"whisper_cpp_binary": self.settings.whisper_cpp_binary,
|
|
592
|
+
"whisper_cpp_language": self.settings.whisper_cpp_language,
|
|
593
|
+
"model_files": model,
|
|
594
|
+
"ocr_enabled": self.settings.ocr_enabled,
|
|
595
|
+
"ocr_model_dir": self.settings.ocr_model_dir,
|
|
596
|
+
"ocr_files": ocr,
|
|
597
|
+
"languages": self.settings.transcript_languages,
|
|
598
|
+
"frame_budget": self.settings.frame_budget,
|
|
599
|
+
"implementation": _sha256_file(Path(__file__).with_name("runners.py")),
|
|
600
|
+
"extraction_implementation": extraction_implementation_fingerprint(),
|
|
601
|
+
}
|
|
602
|
+
runtime_versions: dict[str, str] = {}
|
|
603
|
+
for distribution in ("faster-whisper", "ctranslate2", "rapidocr", "onnxruntime", "Pillow"):
|
|
604
|
+
try:
|
|
605
|
+
runtime_versions[distribution] = version(distribution)
|
|
606
|
+
except PackageNotFoundError:
|
|
607
|
+
runtime_versions[distribution] = "not-installed"
|
|
608
|
+
binary = shutil.which(self.settings.whisper_cpp_binary) if self.settings.whisper_cpp_binary else None
|
|
609
|
+
payload["runtime_versions"] = runtime_versions
|
|
610
|
+
payload["whisper_cpp_binary_sha256"] = _sha256_file(Path(binary)) if binary is not None else None
|
|
611
|
+
return "sha256:" + hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
|
|
612
|
+
|
|
613
|
+
def _reusable_content(self, output_dir: Path, previous: Any, ref: ContentRef) -> ContentBundle | None:
|
|
614
|
+
"""Reuse a verified platform stage without disguising it as a fixture."""
|
|
615
|
+
cache = getattr(previous, "content_cache", None)
|
|
616
|
+
if not isinstance(cache, dict) or cache.get("origin") != "platform":
|
|
617
|
+
return None
|
|
618
|
+
if cache.get("processing_fingerprint") != self._processing_fingerprint():
|
|
619
|
+
return None
|
|
620
|
+
if any(previous.stages.get(stage) != "complete" for stage in ("acquire", "extract")):
|
|
621
|
+
return None
|
|
622
|
+
try:
|
|
623
|
+
for filename in ("knowledge.json", "content.json", "source.json"):
|
|
624
|
+
path = output_dir / filename
|
|
625
|
+
if path.is_symlink() or _sha256_file(path) != cache.get(filename):
|
|
626
|
+
return None
|
|
627
|
+
bundle = ExtractionBundleV2.model_validate_json((output_dir / "knowledge.json").read_text(encoding="utf-8"))
|
|
628
|
+
if len(bundle.contents) != 1 or len(bundle.sources) != 1:
|
|
629
|
+
return None
|
|
630
|
+
content = bundle.contents[0]
|
|
631
|
+
if (
|
|
632
|
+
not self._same_content_unit(ref, content)
|
|
633
|
+
or previous.source_ids != [content.source.source_id]
|
|
634
|
+
or content.source_revision != previous.source_revision
|
|
635
|
+
or content.completeness.status != "complete"
|
|
636
|
+
):
|
|
637
|
+
return None
|
|
638
|
+
for asset in content.assets:
|
|
639
|
+
path = output_dir / asset.relative_path
|
|
640
|
+
if (
|
|
641
|
+
not path.resolve().is_relative_to(output_dir.resolve())
|
|
642
|
+
or path.is_symlink() or path.stat().st_size != asset.byte_size
|
|
643
|
+
or _sha256_file(path) != asset.sha256
|
|
644
|
+
):
|
|
645
|
+
return None
|
|
646
|
+
return content
|
|
647
|
+
except (OSError, ValueError, TypeError):
|
|
648
|
+
return None
|
|
649
|
+
|
|
650
|
+
def _acquired_content(self, ref: ContentRef, output_dir: Path, previous: Any, access: AccessContext,
|
|
651
|
+
*, refresh: bool, content_type: ContentType | None) -> tuple[ContentBundle, Any]:
|
|
652
|
+
from ytk.pipeline.acquisition_checkpoint import (
|
|
653
|
+
configuration_fingerprint,
|
|
654
|
+
load_checkpoint,
|
|
655
|
+
save_checkpoint,
|
|
656
|
+
)
|
|
657
|
+
|
|
658
|
+
store = self._store(output_dir)
|
|
659
|
+
configuration = configuration_fingerprint(self.settings, self._processing_fingerprint())
|
|
660
|
+
if store is not None and not refresh:
|
|
661
|
+
restored = load_checkpoint(store, previous, ref, configuration,
|
|
662
|
+
same_unit=self._same_content_unit, sanitize=_sanitize_metadata)
|
|
663
|
+
if restored is not None and (content_type is None or restored[0].source.content_type == content_type):
|
|
664
|
+
self._assert_output_ownership(output_dir, previous, restored[0].source)
|
|
665
|
+
return restored
|
|
666
|
+
try:
|
|
667
|
+
acquisition = self._acquire(ref, work_dir=output_dir / "artifacts" / "acquire", access=access, refresh=refresh)
|
|
668
|
+
except Exception as exc:
|
|
669
|
+
converted = self._convert_sidecar_error(exc, output_dir)
|
|
670
|
+
if converted is not None:
|
|
671
|
+
raise converted from exc
|
|
672
|
+
raise
|
|
673
|
+
acquired_ref = getattr(acquisition, "source", None)
|
|
674
|
+
if isinstance(acquired_ref, ContentRef):
|
|
675
|
+
ref = acquired_ref
|
|
676
|
+
self._assert_output_ownership(output_dir, previous, ref)
|
|
677
|
+
content = self._content_from_acquisition(ref, acquisition, output_dir)
|
|
678
|
+
if store is not None:
|
|
679
|
+
save_checkpoint(store, content, acquisition, configuration, sanitize=_sanitize_metadata)
|
|
680
|
+
return content, acquisition
|
|
681
|
+
|
|
682
|
+
@staticmethod
|
|
683
|
+
def _same_content_unit(ref: ContentRef, content: ContentBundle) -> bool:
|
|
684
|
+
if ref.source_id == content.source.source_id:
|
|
685
|
+
return True
|
|
686
|
+
if ref.platform != "bilibili" or ref.variant_id is not None:
|
|
687
|
+
return False
|
|
688
|
+
if ref.platform != content.source.platform or ref.content_id != content.source.content_id:
|
|
689
|
+
return False
|
|
690
|
+
from urllib.parse import parse_qs, urlsplit
|
|
691
|
+
|
|
692
|
+
try:
|
|
693
|
+
requested_part = int(ref.display.get("p") or 1)
|
|
694
|
+
saved_part = content.metadata.get("p") or content.source.display.get("p")
|
|
695
|
+
if saved_part is None:
|
|
696
|
+
saved_part = parse_qs(urlsplit(content.source.canonical_url).query).get("p", [1])[0]
|
|
697
|
+
return requested_part == int(str(saved_part))
|
|
698
|
+
except (ValueError, TypeError):
|
|
699
|
+
return False
|
|
700
|
+
|
|
701
|
+
def _content_from_fixture(self, fixture: FixtureContent, output_dir: Path) -> ContentBundle:
|
|
702
|
+
source_id = fixture.source.source_id
|
|
703
|
+
blocks = list(fixture.blocks)
|
|
704
|
+
if fixture.subtitle_text:
|
|
705
|
+
blocks.extend(self._blocks_from_subtitle_text(fixture.subtitle_text, source_id, start_order=len(blocks)))
|
|
706
|
+
assets = self._copy_fixture_media(fixture.media, source_id, output_dir)
|
|
707
|
+
fingerprint = {
|
|
708
|
+
"blocks": [block.block_id for block in blocks],
|
|
709
|
+
"assets": [asset.sha256 for asset in assets],
|
|
710
|
+
}
|
|
711
|
+
return ContentBundle(
|
|
712
|
+
source=fixture.source,
|
|
713
|
+
source_revision=compute_source_revision(source_id, fingerprint),
|
|
714
|
+
assets=assets,
|
|
715
|
+
blocks=blocks,
|
|
716
|
+
completeness=fixture.completeness,
|
|
717
|
+
provider_trace=fixture.provider_trace,
|
|
718
|
+
metadata=dict(fixture.metadata),
|
|
719
|
+
)
|
|
720
|
+
|
|
721
|
+
def _browser_content_from_fixture(self, fixture: FixtureContent, output_dir: Path, url: str) -> tuple[ContentBundle, Any]:
|
|
722
|
+
"""Reject bad capture descriptors before touching a previous delivery."""
|
|
723
|
+
from ytk.ingestion.browser_acquired import verify_browser_acquisition
|
|
724
|
+
if len(fixture.media) != 1:
|
|
725
|
+
raise UserFacingError("browser acquisition requires one actual MP4 video")
|
|
726
|
+
local = fixture.media[0].get("local_path")
|
|
727
|
+
if (not isinstance(local, str) or Path(local).suffix.lower() != ".mp4"
|
|
728
|
+
or Path(local).is_symlink() or fixture.media[0].get("media_type") != "video"):
|
|
729
|
+
raise UserFacingError("browser acquisition requires a regular local MP4 video")
|
|
730
|
+
with tempfile.TemporaryDirectory(prefix=".kd-browser-verify-", dir=output_dir.parent) as work:
|
|
731
|
+
stage = Path(work)
|
|
732
|
+
content = self._content_from_fixture(fixture, stage)
|
|
733
|
+
content, acquisition = verify_browser_acquisition(content, stage, url)
|
|
734
|
+
asset = content.assets[0]
|
|
735
|
+
relative = f"assets/browser-{asset.sha256}.mp4"
|
|
736
|
+
target = output_dir / relative
|
|
737
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
738
|
+
_copy_current_asset(stage / asset.relative_path, target)
|
|
739
|
+
published = asset.model_copy(update={"relative_path": relative})
|
|
740
|
+
content = content.model_copy(update={"assets": [published]})
|
|
741
|
+
acquisition = acquisition.model_copy(update={"media": [acquisition.media[0].model_copy(update={"local_path": str(target)})]})
|
|
742
|
+
return content, acquisition
|
|
743
|
+
|
|
744
|
+
def _content_from_acquisition(self, ref: ContentRef, acquisition: Any, output_dir: Path) -> ContentBundle:
|
|
745
|
+
source_id = ref.source_id
|
|
746
|
+
media = list(getattr(acquisition, "media", []) or [])
|
|
747
|
+
assets = list(getattr(acquisition, "assets", []) or [])
|
|
748
|
+
if not assets and media:
|
|
749
|
+
assets = self._materialize_media(media, source_id, output_dir)
|
|
750
|
+
blocks = self._blocks_from_acquisition(acquisition, source_id)
|
|
751
|
+
completeness = getattr(acquisition, "completeness", None) or ContentCompleteness(
|
|
752
|
+
status="unknown", reason_code=getattr(acquisition, "reason_code", None)
|
|
753
|
+
)
|
|
754
|
+
fingerprint = {
|
|
755
|
+
"blocks": [block.block_id for block in blocks],
|
|
756
|
+
"assets": [asset.sha256 for asset in assets],
|
|
757
|
+
"metadata_title": str(getattr(acquisition, "metadata", {}).get("title", "")),
|
|
758
|
+
}
|
|
759
|
+
return ContentBundle(
|
|
760
|
+
source=ref,
|
|
761
|
+
source_revision=compute_source_revision(source_id, fingerprint),
|
|
762
|
+
assets=assets,
|
|
763
|
+
blocks=blocks,
|
|
764
|
+
completeness=completeness,
|
|
765
|
+
provider_trace=list(getattr(acquisition, "provider_trace", []) or []),
|
|
766
|
+
metadata=dict(getattr(acquisition, "metadata", {}) or {}),
|
|
767
|
+
)
|
|
768
|
+
|
|
769
|
+
@staticmethod
|
|
770
|
+
def _blocks_from_acquisition(acquisition: Any, source_id: str) -> list[ContentBlock]:
|
|
771
|
+
"""Turn adapter ``AcquiredBlock`` items into locator-keyed content blocks."""
|
|
772
|
+
blocks: list[ContentBlock] = []
|
|
773
|
+
seen: set[str] = set()
|
|
774
|
+
for raw in getattr(acquisition, "blocks", []) or []:
|
|
775
|
+
locator = raw.locator
|
|
776
|
+
raw_text = raw.raw_text
|
|
777
|
+
block_id = make_block_id(source_id, locator, raw_text)
|
|
778
|
+
if block_id in seen:
|
|
779
|
+
continue
|
|
780
|
+
seen.add(block_id)
|
|
781
|
+
blocks.append(
|
|
782
|
+
ContentBlock(
|
|
783
|
+
block_id=block_id,
|
|
784
|
+
source_id=source_id,
|
|
785
|
+
kind=raw.kind,
|
|
786
|
+
raw_text=raw_text,
|
|
787
|
+
cleaned_text=getattr(raw, "cleaned_text", None),
|
|
788
|
+
language=getattr(raw, "language", None),
|
|
789
|
+
order=int(getattr(raw, "order", len(blocks))),
|
|
790
|
+
locator=locator,
|
|
791
|
+
)
|
|
792
|
+
)
|
|
793
|
+
return blocks
|
|
794
|
+
|
|
795
|
+
def _copy_fixture_media(
|
|
796
|
+
self, media: Sequence[Mapping[str, object]], source_id: str, output_dir: Path
|
|
797
|
+
) -> list[MediaAsset]:
|
|
798
|
+
assets: list[MediaAsset] = []
|
|
799
|
+
assets_dir = output_dir / "assets"
|
|
800
|
+
for index, entry in enumerate(media):
|
|
801
|
+
local = entry.get("local_path")
|
|
802
|
+
if not isinstance(local, str):
|
|
803
|
+
continue
|
|
804
|
+
source_path = Path(local)
|
|
805
|
+
if not source_path.is_file():
|
|
806
|
+
raise UserFacingError(f"fixture media not found: {source_path}")
|
|
807
|
+
assets_dir.mkdir(parents=True, exist_ok=True)
|
|
808
|
+
media_type = str(entry.get("media_type") or "image")
|
|
809
|
+
suffix = source_path.suffix or ".bin"
|
|
810
|
+
relative = f"assets/{media_type}-{index:03d}{suffix}"
|
|
811
|
+
target = output_dir / relative
|
|
812
|
+
_copy_current_asset(source_path, target)
|
|
813
|
+
assets.append(
|
|
814
|
+
MediaAsset(
|
|
815
|
+
asset_id=f"asset-{index:04d}",
|
|
816
|
+
source_id=source_id,
|
|
817
|
+
relative_path=relative,
|
|
818
|
+
media_type=media_type, # type: ignore[arg-type]
|
|
819
|
+
order=int(entry.get("order", index)), # type: ignore[call-overload]
|
|
820
|
+
byte_size=target.stat().st_size,
|
|
821
|
+
sha256=_sha256_file(target),
|
|
822
|
+
)
|
|
823
|
+
)
|
|
824
|
+
return assets
|
|
825
|
+
|
|
826
|
+
def _materialize_media(
|
|
827
|
+
self, media: Sequence[Any], source_id: str, output_dir: Path
|
|
828
|
+
) -> list[MediaAsset]:
|
|
829
|
+
"""Download/attach media descriptors produced by an adapter."""
|
|
830
|
+
assets: list[MediaAsset] = []
|
|
831
|
+
assets_dir = output_dir / "assets"
|
|
832
|
+
for index, descriptor in enumerate(media):
|
|
833
|
+
local_path = getattr(descriptor, "local_path", None)
|
|
834
|
+
if not local_path:
|
|
835
|
+
continue
|
|
836
|
+
source_path = Path(local_path)
|
|
837
|
+
if not source_path.is_file():
|
|
838
|
+
continue
|
|
839
|
+
assets_dir.mkdir(parents=True, exist_ok=True)
|
|
840
|
+
media_type = getattr(descriptor, "media_type", "image")
|
|
841
|
+
relative = f"assets/{media_type}-{index:03d}{source_path.suffix or '.bin'}"
|
|
842
|
+
target = output_dir / relative
|
|
843
|
+
_copy_current_asset(source_path, target)
|
|
844
|
+
assets.append(
|
|
845
|
+
MediaAsset(
|
|
846
|
+
asset_id=f"asset-{index:04d}",
|
|
847
|
+
source_id=source_id,
|
|
848
|
+
relative_path=relative,
|
|
849
|
+
media_type=cast(Literal["video", "audio", "image", "frame"], media_type),
|
|
850
|
+
order=int(getattr(descriptor, "order", index)),
|
|
851
|
+
byte_size=target.stat().st_size,
|
|
852
|
+
sha256=_sha256_file(target),
|
|
853
|
+
)
|
|
854
|
+
)
|
|
855
|
+
return assets
|
|
856
|
+
|
|
857
|
+
def _blocks_from_subtitle_text(self, text: str, source_id: str, *, start_order: int = 0) -> list[ContentBlock]:
|
|
858
|
+
from ytk.ingestion.adapters.base import SubtitleDescriptor
|
|
859
|
+
from ytk.ingestion.subtitles import parse_subtitle
|
|
860
|
+
from ytk.models.content import ParagraphLocator
|
|
861
|
+
|
|
862
|
+
if "-->" in text or text.lstrip().startswith("WEBVTT"):
|
|
863
|
+
parsed = parse_subtitle(
|
|
864
|
+
SubtitleDescriptor(kind="inline", format="vtt" if text.lstrip().startswith("WEBVTT") else "srt", inline_data=text),
|
|
865
|
+
source_id=source_id,
|
|
866
|
+
)
|
|
867
|
+
return [ContentBlock(
|
|
868
|
+
block_id=make_block_id(source_id, block.locator, block.raw_text),
|
|
869
|
+
source_id=source_id, kind="caption", raw_text=block.raw_text,
|
|
870
|
+
order=start_order + index, locator=block.locator,
|
|
871
|
+
) for index, block in enumerate(parsed.blocks)]
|
|
872
|
+
|
|
873
|
+
blocks: list[ContentBlock] = []
|
|
874
|
+
for index, line in enumerate(text.splitlines()):
|
|
875
|
+
stripped = line.strip()
|
|
876
|
+
if not stripped:
|
|
877
|
+
continue
|
|
878
|
+
locator = ParagraphLocator(paragraph=index)
|
|
879
|
+
blocks.append(
|
|
880
|
+
ContentBlock(
|
|
881
|
+
block_id=make_block_id(source_id, locator, stripped),
|
|
882
|
+
source_id=source_id,
|
|
883
|
+
kind="caption",
|
|
884
|
+
raw_text=stripped,
|
|
885
|
+
order=start_order + index,
|
|
886
|
+
locator=locator,
|
|
887
|
+
)
|
|
888
|
+
)
|
|
889
|
+
return blocks
|
|
890
|
+
|
|
891
|
+
def _maybe_frame_ocr(self, content: ContentBundle, output_dir: Path) -> ContentBundle:
|
|
892
|
+
"""Budget-bound OCR of sampled video frames for videos without narration (T32.2).
|
|
893
|
+
|
|
894
|
+
Only runs when the engine is enabled, a local video asset exists and no
|
|
895
|
+
caption/transcript timed blocks exist (narration evidence wins).
|
|
896
|
+
"""
|
|
897
|
+
if not bool(getattr(self.settings, "ocr_enabled", False)):
|
|
898
|
+
return content
|
|
899
|
+
max_frames = self.settings.frame_budget
|
|
900
|
+
if max_frames <= 0:
|
|
901
|
+
return content
|
|
902
|
+
if self.ocr_runner is None:
|
|
903
|
+
return content
|
|
904
|
+
video_assets = [asset for asset in content.assets if asset.media_type == "video"]
|
|
905
|
+
if not video_assets:
|
|
906
|
+
return content
|
|
907
|
+
has_narration = any(
|
|
908
|
+
block.kind in {"caption", "transcript"} and block.locator.kind == "time_range"
|
|
909
|
+
for block in content.blocks
|
|
910
|
+
)
|
|
911
|
+
if has_narration:
|
|
912
|
+
return content
|
|
913
|
+
try:
|
|
914
|
+
from ytk.ingestion.media import MediaService
|
|
915
|
+
from ytk.ocr.engine import get_engine
|
|
916
|
+
from ytk.ocr.pipeline import run_ocr_for_frames
|
|
917
|
+
|
|
918
|
+
service = MediaService(settings=self.settings)
|
|
919
|
+
source_path = Path(output_dir) / video_assets[0].relative_path
|
|
920
|
+
if not source_path.is_file():
|
|
921
|
+
return content
|
|
922
|
+
frames_dir = Path(output_dir) / "artifacts" / "frames"
|
|
923
|
+
budget = {"max_frames": max_frames}
|
|
924
|
+
frames = service.sample_frames(source_path, frames_dir, budget=budget)
|
|
925
|
+
if not frames:
|
|
926
|
+
return content
|
|
927
|
+
frame_refs = [(float(frame.time), frame.path) for frame in frames]
|
|
928
|
+
start_order = max((block.order for block in content.blocks), default=-1) + 1
|
|
929
|
+
blocks = run_ocr_for_frames(
|
|
930
|
+
frame_refs,
|
|
931
|
+
source_id=content.source.source_id,
|
|
932
|
+
start_order=start_order,
|
|
933
|
+
frame_budget=budget,
|
|
934
|
+
engine=get_engine(model_dir=str(self.settings.resolved_ocr_model_dir())),
|
|
935
|
+
)
|
|
936
|
+
except ModelUnavailableError:
|
|
937
|
+
raise
|
|
938
|
+
except Exception as exc: # frame OCR is enhancement: degrade, never crash
|
|
939
|
+
LOGGER.warning("video frame OCR failed for %s: %s", content.source.source_id, exc)
|
|
940
|
+
return content
|
|
941
|
+
if not blocks:
|
|
942
|
+
return content
|
|
943
|
+
existing = {block.block_id for block in content.blocks}
|
|
944
|
+
merged = list(content.blocks) + [block for block in blocks if block.block_id not in existing]
|
|
945
|
+
return content.model_copy(update={"blocks": merged})
|
|
946
|
+
|
|
947
|
+
# ------------------------------------------------------------ enrichment
|
|
948
|
+
def _maybe_ocr(self, content: ContentBundle, output_dir: Path) -> ContentBundle:
|
|
949
|
+
if content.source.content_type != "image_note":
|
|
950
|
+
return content
|
|
951
|
+
if self.ocr_runner is None:
|
|
952
|
+
image_count = sum(asset.media_type == "image" for asset in content.assets)
|
|
953
|
+
recognized = {
|
|
954
|
+
block.locator.image_index for block in content.blocks
|
|
955
|
+
if block.kind == "ocr" and block.locator.kind == "image_region"
|
|
956
|
+
}
|
|
957
|
+
if not set(range(1, image_count + 1)) <= recognized:
|
|
958
|
+
return content.model_copy(update={"completeness": ContentCompleteness(
|
|
959
|
+
status="partial", checked=list(content.completeness.checked),
|
|
960
|
+
missing=list(dict.fromkeys([*content.completeness.missing, "ocr"])),
|
|
961
|
+
reason_code="OCR_DISABLED" if not self.settings.ocr_enabled else "OCR_UNAVAILABLE",
|
|
962
|
+
)})
|
|
963
|
+
return content
|
|
964
|
+
try:
|
|
965
|
+
blocks, ocr_metadata = _split_runner_output(self.ocr_runner(content, output_dir))
|
|
966
|
+
except Exception as exc: # OCR unavailable/one image failed → PARTIAL, never crash
|
|
967
|
+
LOGGER.warning("OCR stage failed for %s: %s", content.source.source_id, exc)
|
|
968
|
+
return content.model_copy(
|
|
969
|
+
update={
|
|
970
|
+
"completeness": ContentCompleteness(
|
|
971
|
+
status="partial",
|
|
972
|
+
checked=list(content.completeness.checked) + ["ocr"],
|
|
973
|
+
missing=["ocr"],
|
|
974
|
+
reason_code="OCR_UNAVAILABLE",
|
|
975
|
+
)
|
|
976
|
+
}
|
|
977
|
+
)
|
|
978
|
+
existing = {block.block_id for block in content.blocks}
|
|
979
|
+
merged = list(content.blocks) + [block for block in blocks if block.block_id not in existing]
|
|
980
|
+
metadata = dict(content.metadata)
|
|
981
|
+
completeness = content.completeness
|
|
982
|
+
if ocr_metadata:
|
|
983
|
+
metadata["ocr"] = ocr_metadata
|
|
984
|
+
if not ocr_metadata.get("complete", True):
|
|
985
|
+
completeness = ContentCompleteness(
|
|
986
|
+
status="partial",
|
|
987
|
+
checked=list(content.completeness.checked) + ["ocr"],
|
|
988
|
+
missing=list(dict.fromkeys([*content.completeness.missing, "ocr"])),
|
|
989
|
+
reason_code="OCR_INCOMPLETE",
|
|
990
|
+
)
|
|
991
|
+
return content.model_copy(update={"blocks": merged, "metadata": metadata, "completeness": completeness})
|
|
992
|
+
|
|
993
|
+
def _maybe_asr(self, content: ContentBundle, output_dir: Path) -> ContentBundle:
|
|
994
|
+
if self.asr_runner is None:
|
|
995
|
+
return content
|
|
996
|
+
probe = content.metadata.get("media_probe")
|
|
997
|
+
if content.metadata.get("acquisition_mode") == "user_import" and isinstance(probe, dict) and probe.get("audio_streams") == 0:
|
|
998
|
+
return content
|
|
999
|
+
has_timed = any(
|
|
1000
|
+
block.locator.kind == "time_range" and block.kind in {"caption", "transcript"}
|
|
1001
|
+
for block in content.blocks
|
|
1002
|
+
)
|
|
1003
|
+
has_audio = any(asset.media_type in {"audio", "video"} for asset in content.assets)
|
|
1004
|
+
if has_timed or not has_audio:
|
|
1005
|
+
return content
|
|
1006
|
+
try:
|
|
1007
|
+
asr_output = self.asr_runner(content, output_dir)
|
|
1008
|
+
except ServiceNotConfiguredError:
|
|
1009
|
+
raise
|
|
1010
|
+
except ModelUnavailableError:
|
|
1011
|
+
raise
|
|
1012
|
+
except Exception as exc:
|
|
1013
|
+
LOGGER.warning("ASR stage failed for %s: %s", content.source.source_id, exc)
|
|
1014
|
+
return content.model_copy(
|
|
1015
|
+
update={
|
|
1016
|
+
"completeness": ContentCompleteness(
|
|
1017
|
+
status="partial",
|
|
1018
|
+
checked=list(content.completeness.checked) + ["asr"],
|
|
1019
|
+
missing=["asr"],
|
|
1020
|
+
reason_code="ASR_UNAVAILABLE",
|
|
1021
|
+
)
|
|
1022
|
+
}
|
|
1023
|
+
)
|
|
1024
|
+
blocks, asr_metadata = _split_runner_output(asr_output)
|
|
1025
|
+
existing = {block.block_id for block in content.blocks}
|
|
1026
|
+
merged = list(content.blocks) + [block for block in blocks if block.block_id not in existing]
|
|
1027
|
+
metadata = dict(content.metadata)
|
|
1028
|
+
if asr_metadata:
|
|
1029
|
+
metadata["asr"] = asr_metadata
|
|
1030
|
+
metadata["transcript_source"] = "asr"
|
|
1031
|
+
return content.model_copy(update={"blocks": merged, "metadata": metadata})
|
|
1032
|
+
|
|
1033
|
+
@staticmethod
|
|
1034
|
+
def _assess_completeness(acquisition: Any, content: ContentBundle) -> ContentBundle:
|
|
1035
|
+
try:
|
|
1036
|
+
from ytk.quality.completeness import assess
|
|
1037
|
+
|
|
1038
|
+
assessment = assess(acquisition, content)
|
|
1039
|
+
except Exception as exc: # assessment must never crash the run
|
|
1040
|
+
LOGGER.warning("completeness assessment failed: %s", exc)
|
|
1041
|
+
return content
|
|
1042
|
+
if assessment.content == content.completeness:
|
|
1043
|
+
return content
|
|
1044
|
+
return content.model_copy(update={"completeness": assessment.content})
|
|
1045
|
+
|
|
1046
|
+
# ------------------------------------------------------------ assembly
|
|
1047
|
+
def _assemble(self, ref: ContentRef, content: ContentBundle) -> ExtractionBundleV2:
|
|
1048
|
+
source_id = ref.source_id
|
|
1049
|
+
cleaned = self.cleaning.clean_blocks(content.blocks)
|
|
1050
|
+
ordered = sorted(cleaned, key=lambda block: block.order)
|
|
1051
|
+
chapters = content.metadata.get("chapters")
|
|
1052
|
+
chapter_list = chapters if isinstance(chapters, list) else None
|
|
1053
|
+
topics = self.segmentation.segment_blocks(ordered, chapters=chapter_list)
|
|
1054
|
+
extraction: BlockExtraction = self.extraction.extract_blocks(
|
|
1055
|
+
ordered,
|
|
1056
|
+
source_id=source_id,
|
|
1057
|
+
source_revision=content.source_revision,
|
|
1058
|
+
topic_groups=topics,
|
|
1059
|
+
block_processors=self._block_processors(ordered, content),
|
|
1060
|
+
)
|
|
1061
|
+
metadata = dict(content.metadata)
|
|
1062
|
+
metadata.setdefault("title", ref.display.get("title") or source_id)
|
|
1063
|
+
metadata.setdefault("canonical_url", ref.canonical_url)
|
|
1064
|
+
metadata.setdefault("platform", ref.platform)
|
|
1065
|
+
metadata.setdefault("source_id", source_id)
|
|
1066
|
+
atoms = list(extraction.atoms)
|
|
1067
|
+
evidence = dict(extraction.evidence)
|
|
1068
|
+
if not atoms and self.coarse_atoms:
|
|
1069
|
+
atoms, coarse_evidence = self._coarse_atoms(
|
|
1070
|
+
ordered, topics, evidence, source_id=source_id,
|
|
1071
|
+
source_revision=content.source_revision,
|
|
1072
|
+
block_processors=self._block_processors(ordered, content),
|
|
1073
|
+
)
|
|
1074
|
+
evidence.update(coarse_evidence)
|
|
1075
|
+
if atoms:
|
|
1076
|
+
metadata = {**metadata, "coarse_fallback": "segment_summary"}
|
|
1077
|
+
LOGGER.warning(
|
|
1078
|
+
"no structured atoms; emitted %d coarse segment summaries "
|
|
1079
|
+
"(confidence 0.25, explicitly labelled)", len(atoms)
|
|
1080
|
+
)
|
|
1081
|
+
return ExtractionBundleV2(
|
|
1082
|
+
sources=[ref],
|
|
1083
|
+
contents=[content.model_copy(update={"blocks": ordered})],
|
|
1084
|
+
evidence_index=evidence,
|
|
1085
|
+
topics=list(topics),
|
|
1086
|
+
knowledge=atoms,
|
|
1087
|
+
metadata=metadata,
|
|
1088
|
+
skill_spec=None,
|
|
1089
|
+
)
|
|
1090
|
+
|
|
1091
|
+
@staticmethod
|
|
1092
|
+
def _coarse_atoms(
|
|
1093
|
+
blocks: Sequence[ContentBlock],
|
|
1094
|
+
topics: Sequence[Any],
|
|
1095
|
+
evidence: dict[str, EvidenceRefV2],
|
|
1096
|
+
*,
|
|
1097
|
+
source_id: str,
|
|
1098
|
+
source_revision: str,
|
|
1099
|
+
block_processors: Mapping[str, str],
|
|
1100
|
+
) -> tuple[list[KnowledgeAtomV2], dict[str, EvidenceRefV2]]:
|
|
1101
|
+
"""每个 topic 一条低置信分段摘要。
|
|
1102
|
+
|
|
1103
|
+
只在显式开启 coarse 模式且完全抽不出原子时调用。产物带
|
|
1104
|
+
``metadata.coarse=True`` 与 0.25 置信度,``explanation`` 明写它是
|
|
1105
|
+
分段摘要而不是抽取到的知识——允许"弱结论",不允许"假结论"。
|
|
1106
|
+
"""
|
|
1107
|
+
# 证据必须来自 block 本身:即使走粗粒度兜底,也不允许出现无定位的原子。
|
|
1108
|
+
by_block: dict[str, list[str]] = {}
|
|
1109
|
+
for ev_id, ref in evidence.items():
|
|
1110
|
+
by_block.setdefault(getattr(ref, "block_id", ""), []).append(ev_id)
|
|
1111
|
+
text_of = {b.block_id: (b.raw_text or "").strip() for b in blocks}
|
|
1112
|
+
block_by_id = {b.block_id: b for b in blocks}
|
|
1113
|
+
new_evidence: dict[str, EvidenceRefV2] = {}
|
|
1114
|
+
|
|
1115
|
+
def ensure_evidence(block_id: str) -> list[str]:
|
|
1116
|
+
existing = by_block.get(block_id)
|
|
1117
|
+
if existing:
|
|
1118
|
+
return existing
|
|
1119
|
+
block = block_by_id.get(block_id)
|
|
1120
|
+
if block is None:
|
|
1121
|
+
return []
|
|
1122
|
+
ev_id = "%s@%s#%s#%04d" % (source_id, source_revision, block_id,
|
|
1123
|
+
len(new_evidence) + 1)
|
|
1124
|
+
# 粗粒度兜底拿不准来源时,标记为 unknown 而不是谎称 subtitle_auto
|
|
1125
|
+
processor = (block_processors or {}).get(block.block_id) or "unknown"
|
|
1126
|
+
new_evidence[ev_id] = EvidenceRefV2(
|
|
1127
|
+
evidence_id=ev_id,
|
|
1128
|
+
source_id=source_id,
|
|
1129
|
+
source_revision=source_revision,
|
|
1130
|
+
block_id=block.block_id,
|
|
1131
|
+
quote=(block.raw_text or "").strip()[:500],
|
|
1132
|
+
locator=block.locator,
|
|
1133
|
+
processor=processor, # type: ignore[arg-type]
|
|
1134
|
+
)
|
|
1135
|
+
by_block.setdefault(block_id, []).append(ev_id)
|
|
1136
|
+
return [ev_id]
|
|
1137
|
+
|
|
1138
|
+
atoms: list[KnowledgeAtomV2] = []
|
|
1139
|
+
for index, topic in enumerate(topics, start=1):
|
|
1140
|
+
block_ids = [b for b in getattr(topic, "block_ids", []) if b in text_of]
|
|
1141
|
+
joined = " ".join(text_of[b] for b in block_ids).strip()
|
|
1142
|
+
ev_ids = [e for b in block_ids for e in ensure_evidence(b)]
|
|
1143
|
+
if not joined or not ev_ids:
|
|
1144
|
+
continue
|
|
1145
|
+
title = getattr(topic, "title", "") or "segment %d" % index
|
|
1146
|
+
atoms.append(
|
|
1147
|
+
KnowledgeAtomV2(
|
|
1148
|
+
atom_id="coarse-%d" % index,
|
|
1149
|
+
atom_type="segment_summary",
|
|
1150
|
+
content=joined[:400],
|
|
1151
|
+
explanation=(
|
|
1152
|
+
"低置信分段摘要:本段未抽取出结构化知识,这里只是按主题切出的"
|
|
1153
|
+
"原文摘录,不能当作结论使用。"
|
|
1154
|
+
),
|
|
1155
|
+
source_ids=[source_id],
|
|
1156
|
+
evidence_ids=ev_ids,
|
|
1157
|
+
confidence=0.25,
|
|
1158
|
+
topic_path=[title],
|
|
1159
|
+
metadata={"coarse": True, "block_count": len(block_ids)},
|
|
1160
|
+
)
|
|
1161
|
+
)
|
|
1162
|
+
return atoms, new_evidence
|
|
1163
|
+
|
|
1164
|
+
@staticmethod
|
|
1165
|
+
def _block_processors(blocks: Sequence[ContentBlock], content: ContentBundle) -> dict[str, str]:
|
|
1166
|
+
"""Truthful evidence processors: ASR text is never stamped as a subtitle."""
|
|
1167
|
+
subtitle_meta = content.metadata.get("subtitle")
|
|
1168
|
+
if not isinstance(subtitle_meta, dict):
|
|
1169
|
+
subtitle_meta = {}
|
|
1170
|
+
transcript_source = content.metadata.get("transcript_source")
|
|
1171
|
+
track_kind = subtitle_meta.get("kind")
|
|
1172
|
+
track_format = subtitle_meta.get("format")
|
|
1173
|
+
|
|
1174
|
+
def caption_processor() -> str:
|
|
1175
|
+
if track_kind == "manual":
|
|
1176
|
+
return "subtitle_manual"
|
|
1177
|
+
return {
|
|
1178
|
+
"json3": "subtitle_json3",
|
|
1179
|
+
"srt": "subtitle_srt",
|
|
1180
|
+
"vtt": "subtitle_vtt",
|
|
1181
|
+
}.get(str(track_format or ""), "subtitle_auto")
|
|
1182
|
+
|
|
1183
|
+
processors: dict[str, str] = {}
|
|
1184
|
+
for block in blocks:
|
|
1185
|
+
if content.metadata.get("acquisition_mode") == "user_import" and block.locator.kind == "video_frame":
|
|
1186
|
+
processors[block.block_id] = "visual_review"
|
|
1187
|
+
elif content.metadata.get("transcript_source") == "user_provided" and block.kind == "caption":
|
|
1188
|
+
processors[block.block_id] = "user_provided_subtitle"
|
|
1189
|
+
elif block.kind == "ocr":
|
|
1190
|
+
processors[block.block_id] = "ocr"
|
|
1191
|
+
elif block.kind in {"paragraph", "title", "description"}:
|
|
1192
|
+
processors[block.block_id] = "author_text"
|
|
1193
|
+
elif transcript_source == "asr":
|
|
1194
|
+
processors[block.block_id] = "asr"
|
|
1195
|
+
else:
|
|
1196
|
+
processors[block.block_id] = caption_processor()
|
|
1197
|
+
return processors
|
|
1198
|
+
|
|
1199
|
+
# ------------------------------------------------------------- outcomes
|
|
1200
|
+
def _resolve_formats(self, ref: ContentRef, formats: Sequence[str] | None) -> set[str]:
|
|
1201
|
+
if formats is None:
|
|
1202
|
+
registration = self.registry.get(ref.platform)
|
|
1203
|
+
return set(registration.default_formats)
|
|
1204
|
+
return {value.strip() for value in formats if value and value.strip()}
|
|
1205
|
+
|
|
1206
|
+
def _decide_outcomes(
|
|
1207
|
+
self, bundle: ExtractionBundleV2, requested: set[str]
|
|
1208
|
+
) -> tuple[Outcome, dict[str, str], dict[str, str], list[str]]:
|
|
1209
|
+
reasons: list[str] = []
|
|
1210
|
+
format_status: dict[str, str] = {}
|
|
1211
|
+
format_reasons: dict[str, str] = {}
|
|
1212
|
+
completeness = bundle.contents[0].completeness if bundle.contents else None
|
|
1213
|
+
content_is_complete = completeness is not None and completeness.status == "complete"
|
|
1214
|
+
has_knowledge = bool(bundle.knowledge)
|
|
1215
|
+
has_steps = any(
|
|
1216
|
+
atom.atom_type in {"workflow_step", "terminal_command", "configuration_snippet", "command_sequence", "code_block"}
|
|
1217
|
+
for atom in bundle.knowledge
|
|
1218
|
+
)
|
|
1219
|
+
|
|
1220
|
+
if not has_knowledge:
|
|
1221
|
+
# NO_KNOWLEDGE only when content is provably complete (plan §4.5).
|
|
1222
|
+
if content_is_complete and _no_knowledge_gate(bundle, completeness):
|
|
1223
|
+
for fmt in requested:
|
|
1224
|
+
format_status[fmt] = "pending" if fmt in {"json", "md", "summary"} else "not_applicable"
|
|
1225
|
+
if format_status[fmt] == "not_applicable":
|
|
1226
|
+
format_reasons[fmt] = "NO_KNOWLEDGE"
|
|
1227
|
+
reasons.append("NO_KNOWLEDGE: content complete but no knowledge atoms extracted")
|
|
1228
|
+
return OUTCOME_NO_KNOWLEDGE, format_status, format_reasons, reasons
|
|
1229
|
+
for fmt in requested:
|
|
1230
|
+
format_status[fmt] = "blocked"
|
|
1231
|
+
format_reasons[fmt] = "CONTENT_INCOMPLETE"
|
|
1232
|
+
reasons.append(
|
|
1233
|
+
f"BLOCKED: content incomplete ({completeness.status if completeness else 'unknown'}) "
|
|
1234
|
+
"and no knowledge atoms were extracted"
|
|
1235
|
+
)
|
|
1236
|
+
return OUTCOME_BLOCKED, format_status, format_reasons, reasons
|
|
1237
|
+
|
|
1238
|
+
for fmt in requested:
|
|
1239
|
+
if fmt == "skill" and not has_steps:
|
|
1240
|
+
format_status[fmt] = "not_applicable"
|
|
1241
|
+
format_reasons[fmt] = "OUTPUTS_NOT_APPLICABLE"
|
|
1242
|
+
reasons.append("skill NOT_APPLICABLE: knowledge has no actionable steps")
|
|
1243
|
+
else:
|
|
1244
|
+
format_status[fmt] = "pending"
|
|
1245
|
+
actionable = [fmt for fmt in requested if format_status.get(fmt) == "pending"]
|
|
1246
|
+
if requested and not actionable:
|
|
1247
|
+
reasons.append("FAILED: every requested format is not applicable (OUTPUTS_NOT_APPLICABLE)")
|
|
1248
|
+
return OUTCOME_FAILED, format_status, format_reasons, reasons
|
|
1249
|
+
if completeness is not None and completeness.status != "complete":
|
|
1250
|
+
reasons.append(f"PARTIAL: content completeness={completeness.status}")
|
|
1251
|
+
return OUTCOME_PARTIAL, format_status, format_reasons, reasons
|
|
1252
|
+
return OUTCOME_COMPLETE, format_status, format_reasons, reasons
|
|
1253
|
+
|
|
1254
|
+
# --------------------------------------------------------------- render
|
|
1255
|
+
def _render(
|
|
1256
|
+
self,
|
|
1257
|
+
bundle: ExtractionBundleV2,
|
|
1258
|
+
requested: set[str],
|
|
1259
|
+
output_dir: Path,
|
|
1260
|
+
format_status: dict[str, str],
|
|
1261
|
+
*,
|
|
1262
|
+
doc_language: str | None,
|
|
1263
|
+
) -> dict[str, Path]:
|
|
1264
|
+
written: dict[str, Path] = {}
|
|
1265
|
+
# The canonical v2 product (schema_version=2 JSON) is owned by the
|
|
1266
|
+
# pipeline, not by a pluggable renderer: it is the artifact every other
|
|
1267
|
+
# output and the acceptance validator read back.
|
|
1268
|
+
canonical = self._write_canonical_artifacts(bundle, output_dir)
|
|
1269
|
+
for fmt in sorted(requested):
|
|
1270
|
+
if format_status.get(fmt) in {"not_applicable", "blocked"}:
|
|
1271
|
+
continue
|
|
1272
|
+
if fmt == "json":
|
|
1273
|
+
format_status[fmt] = "complete"
|
|
1274
|
+
written[fmt] = canonical["knowledge"]
|
|
1275
|
+
continue
|
|
1276
|
+
renderer = self.renderers.get(fmt)
|
|
1277
|
+
if renderer is None:
|
|
1278
|
+
format_status[fmt] = "failed"
|
|
1279
|
+
continue
|
|
1280
|
+
target = output_dir / _default_filename(fmt)
|
|
1281
|
+
try:
|
|
1282
|
+
render_kwargs: dict[str, Any] = {"doc_language": doc_language}
|
|
1283
|
+
if fmt == "doc":
|
|
1284
|
+
render_kwargs["allow_cloud"] = self._cloud_text_allowed(bundle)
|
|
1285
|
+
render_kwargs["settings"] = self.settings
|
|
1286
|
+
(output_dir / ".knowledge-doc-status.json").unlink(missing_ok=True)
|
|
1287
|
+
renderer(bundle, target, **render_kwargs)
|
|
1288
|
+
except Exception as exc: # one format failing must not kill the others
|
|
1289
|
+
LOGGER.warning("renderer %s failed: %s", fmt, exc)
|
|
1290
|
+
format_status[fmt] = "failed"
|
|
1291
|
+
continue
|
|
1292
|
+
if fmt == "doc":
|
|
1293
|
+
from ytk.renderers.knowledge_doc_renderer import complete_doc_status
|
|
1294
|
+
|
|
1295
|
+
if not complete_doc_status(output_dir):
|
|
1296
|
+
format_status[fmt] = "blocked"
|
|
1297
|
+
try:
|
|
1298
|
+
status = json.loads((output_dir / ".knowledge-doc-status.json").read_text(encoding="utf-8"))
|
|
1299
|
+
filename = status.get("output_file", "")
|
|
1300
|
+
candidate = output_dir / filename
|
|
1301
|
+
if (
|
|
1302
|
+
isinstance(filename, str) and filename and Path(filename).name == filename
|
|
1303
|
+
and candidate.is_file() and _sha256_file(candidate) == status.get("output_sha256")
|
|
1304
|
+
):
|
|
1305
|
+
written[fmt] = candidate
|
|
1306
|
+
except (OSError, ValueError, TypeError):
|
|
1307
|
+
pass
|
|
1308
|
+
continue
|
|
1309
|
+
format_status[fmt] = "complete"
|
|
1310
|
+
written[fmt] = target
|
|
1311
|
+
return written
|
|
1312
|
+
|
|
1313
|
+
def _write_canonical_artifacts(self, bundle: ExtractionBundleV2, output_dir: Path) -> dict[str, Path]:
|
|
1314
|
+
"""Write schema-v2 JSON artifacts atomically (temp file -> replace)."""
|
|
1315
|
+
knowledge_payload = dump_bundle_json(bundle)
|
|
1316
|
+
source_payload = {
|
|
1317
|
+
"schema_version": 2,
|
|
1318
|
+
"sources": [ref.public_dict() for ref in bundle.sources],
|
|
1319
|
+
"metadata": bundle.metadata,
|
|
1320
|
+
}
|
|
1321
|
+
content_payload = {
|
|
1322
|
+
"schema_version": 2,
|
|
1323
|
+
"contents": [item.model_dump(mode="json") for item in bundle.contents],
|
|
1324
|
+
}
|
|
1325
|
+
store = self._store(output_dir)
|
|
1326
|
+
if store is not None:
|
|
1327
|
+
return {
|
|
1328
|
+
"knowledge": store.write_json("knowledge.json", knowledge_payload),
|
|
1329
|
+
"source": store.write_json("source.json", source_payload),
|
|
1330
|
+
"content": store.write_json("content.json", content_payload),
|
|
1331
|
+
}
|
|
1332
|
+
knowledge = output_dir / "knowledge.json"
|
|
1333
|
+
_atomic_write_json(knowledge, knowledge_payload)
|
|
1334
|
+
source = output_dir / "source.json"
|
|
1335
|
+
_atomic_write_json(source, source_payload)
|
|
1336
|
+
content = output_dir / "content.json"
|
|
1337
|
+
_atomic_write_json(content, content_payload)
|
|
1338
|
+
return {"knowledge": knowledge, "source": source, "content": content}
|
|
1339
|
+
|
|
1340
|
+
def _store(self, output_dir: Path) -> Any | None:
|
|
1341
|
+
factory = self.artifact_store_factory
|
|
1342
|
+
if factory is None:
|
|
1343
|
+
return None
|
|
1344
|
+
try:
|
|
1345
|
+
return factory(output_dir)
|
|
1346
|
+
except Exception as exc: # store is an optimization, never a blocker
|
|
1347
|
+
LOGGER.warning("artifact store unavailable for %s: %s", output_dir, exc)
|
|
1348
|
+
return None
|
|
1349
|
+
|
|
1350
|
+
def _read_previous_manifest(self, output_dir: Path) -> Any | None:
|
|
1351
|
+
store = self._store(output_dir)
|
|
1352
|
+
if store is None:
|
|
1353
|
+
return None
|
|
1354
|
+
try:
|
|
1355
|
+
return store.read_manifest()
|
|
1356
|
+
except Exception as exc: # pragma: no cover - defensive
|
|
1357
|
+
LOGGER.warning("existing manifest unreadable (%s); treating as absent", exc)
|
|
1358
|
+
return None
|
|
1359
|
+
|
|
1360
|
+
@staticmethod
|
|
1361
|
+
def _assert_output_ownership(
|
|
1362
|
+
output_dir: Path, previous: Any | None, ref: ContentRef, *, allow_unresolved_variant: bool = False
|
|
1363
|
+
) -> None:
|
|
1364
|
+
"""Refuse to mix two different works into one output directory (A39/A02/A03).
|
|
1365
|
+
|
|
1366
|
+
Identity comparison is two-tiered: the base (platform:content_id) must
|
|
1367
|
+
match, AND — because a Bilibili ``cid`` denotes a different content unit —
|
|
1368
|
+
the platform variant must match when the existing manifest carries one.
|
|
1369
|
+
A pre-acquire ref without a variant is refused when the directory already
|
|
1370
|
+
holds a variant-bearing record (ambiguous ownership, fail closed).
|
|
1371
|
+
"""
|
|
1372
|
+
if previous is None:
|
|
1373
|
+
return
|
|
1374
|
+
existing = set(getattr(previous, "source_ids", []) or [])
|
|
1375
|
+
if not existing:
|
|
1376
|
+
return
|
|
1377
|
+
|
|
1378
|
+
def _split(source_id: str) -> tuple[str, str | None]:
|
|
1379
|
+
parts = source_id.split(":")
|
|
1380
|
+
return ":".join(parts[:2]), (parts[2] if len(parts) > 2 else None)
|
|
1381
|
+
|
|
1382
|
+
if _split(ref.source_id)[0] not in {_split(item)[0] for item in existing}:
|
|
1383
|
+
raise UserFacingError(
|
|
1384
|
+
f"output directory {output_dir} already belongs to {sorted(existing)}; "
|
|
1385
|
+
f"refusing to write {ref.source_id} into it (use a per-source --out)"
|
|
1386
|
+
)
|
|
1387
|
+
existing_variants = {variant for _, variant in (_split(item) for item in existing) if variant}
|
|
1388
|
+
new_variant = _split(ref.source_id)[1]
|
|
1389
|
+
if allow_unresolved_variant and ref.platform == "bilibili" and new_variant is None:
|
|
1390
|
+
return # cid is checked after acquisition, before any canonical/media write
|
|
1391
|
+
if existing_variants and (new_variant is None or new_variant not in existing_variants):
|
|
1392
|
+
raise UserFacingError(
|
|
1393
|
+
f"output directory {output_dir} already holds variant(s) {sorted(existing_variants)} of "
|
|
1394
|
+
f"the same work; refusing to overwrite with {ref.source_id} "
|
|
1395
|
+
f"(different content unit — use a per-part --out)"
|
|
1396
|
+
)
|
|
1397
|
+
|
|
1398
|
+
@staticmethod
|
|
1399
|
+
def _assert_no_unresolved_remote(previous: Any | None) -> None:
|
|
1400
|
+
"""Never re-dispatch a task whose remote submission state is unknown (A36)."""
|
|
1401
|
+
if previous is None:
|
|
1402
|
+
return
|
|
1403
|
+
unresolved: dict[str, dict[str, Any]] = {}
|
|
1404
|
+
try:
|
|
1405
|
+
from ytk.pipeline.state import unresolved_remote_tasks
|
|
1406
|
+
|
|
1407
|
+
unresolved = dict(unresolved_remote_tasks(previous))
|
|
1408
|
+
except Exception: # pragma: no cover - defensive
|
|
1409
|
+
return
|
|
1410
|
+
unresolved = {
|
|
1411
|
+
provider: record for provider, record in unresolved.items()
|
|
1412
|
+
if not (
|
|
1413
|
+
provider == "dtk" and record.get("task_id")
|
|
1414
|
+
and all(source_id.startswith("douyin:") for source_id in previous.source_ids)
|
|
1415
|
+
and previous.source_ids
|
|
1416
|
+
)
|
|
1417
|
+
}
|
|
1418
|
+
if unresolved:
|
|
1419
|
+
from ytk.errors import AnalysisBlockedError
|
|
1420
|
+
|
|
1421
|
+
raise AnalysisBlockedError(
|
|
1422
|
+
"REMOTE_SUBMISSION_UNKNOWN: this task has an unresolved remote submission "
|
|
1423
|
+
f"({sorted(unresolved)}); refusing to re-dispatch. Resolve or clear it "
|
|
1424
|
+
"explicitly before refreshing."
|
|
1425
|
+
)
|
|
1426
|
+
|
|
1427
|
+
@staticmethod
|
|
1428
|
+
def _remove_stale_artifacts(previous: Any | None, written: Mapping[str, Path], output_dir: Path) -> None:
|
|
1429
|
+
"""Delete artifacts recorded by an older manifest that this run superseded (R-12)."""
|
|
1430
|
+
if previous is None:
|
|
1431
|
+
return
|
|
1432
|
+
keep = {path.resolve() for path in written.values()}
|
|
1433
|
+
for fmt, entry in (getattr(previous, "outputs", {}) or {}).items():
|
|
1434
|
+
if fmt == "doc" and fmt in written:
|
|
1435
|
+
continue
|
|
1436
|
+
recorded = getattr(entry, "path", None)
|
|
1437
|
+
if not recorded:
|
|
1438
|
+
continue
|
|
1439
|
+
candidate = (output_dir / recorded).resolve()
|
|
1440
|
+
if candidate in keep or not candidate.is_file():
|
|
1441
|
+
continue
|
|
1442
|
+
try:
|
|
1443
|
+
if candidate.is_relative_to(output_dir.resolve()):
|
|
1444
|
+
candidate.unlink()
|
|
1445
|
+
LOGGER.info("removed stale artifact %s (was %s)", candidate.name, fmt)
|
|
1446
|
+
except OSError as exc: # pragma: no cover - defensive
|
|
1447
|
+
LOGGER.warning("could not remove stale artifact %s: %s", candidate, exc)
|
|
1448
|
+
|
|
1449
|
+
def _sanitize_bundle(self, bundle: ExtractionBundleV2) -> ExtractionBundleV2:
|
|
1450
|
+
"""Strip credentials/access tokens from everything that will be persisted (A24).
|
|
1451
|
+
|
|
1452
|
+
Redaction is two-layered (adversarial review P0): key-name based — any
|
|
1453
|
+
metadata key that names a credential (SESSDATA, signature, xsec_token …)
|
|
1454
|
+
loses its value — and value-shape based via :func:`ytk.logging.redact_payload`
|
|
1455
|
+
for URL/token patterns anywhere else.
|
|
1456
|
+
"""
|
|
1457
|
+
sanitized_metadata = _sanitize_metadata(dict(bundle.metadata or {}))
|
|
1458
|
+
contents = [
|
|
1459
|
+
content.model_copy(update={"metadata": _sanitize_metadata(dict(content.metadata or {}))})
|
|
1460
|
+
for content in bundle.contents
|
|
1461
|
+
]
|
|
1462
|
+
return bundle.model_copy(
|
|
1463
|
+
update={
|
|
1464
|
+
"metadata": sanitized_metadata if isinstance(sanitized_metadata, dict) else {},
|
|
1465
|
+
"contents": contents,
|
|
1466
|
+
}
|
|
1467
|
+
)
|
|
1468
|
+
|
|
1469
|
+
@staticmethod
|
|
1470
|
+
def _adjust_outcome_after_render(outcome: str, format_status: Mapping[str, str], requested: set[str]) -> str:
|
|
1471
|
+
"""A failed renderer must downgrade the outcome (never COMPLETE with a red format)."""
|
|
1472
|
+
actionable = [fmt for fmt in requested if format_status.get(fmt) not in {"not_applicable", "blocked"}]
|
|
1473
|
+
blocked = [fmt for fmt in requested if format_status.get(fmt) == "blocked"]
|
|
1474
|
+
if not actionable:
|
|
1475
|
+
return OUTCOME_BLOCKED if blocked and outcome == OUTCOME_COMPLETE else outcome
|
|
1476
|
+
complete = [fmt for fmt in actionable if format_status.get(fmt) == "complete"]
|
|
1477
|
+
failed = [fmt for fmt in actionable if format_status.get(fmt) == "failed"]
|
|
1478
|
+
if blocked and not failed and outcome == OUTCOME_COMPLETE:
|
|
1479
|
+
return OUTCOME_PARTIAL if complete else OUTCOME_BLOCKED
|
|
1480
|
+
if not failed:
|
|
1481
|
+
return outcome
|
|
1482
|
+
if outcome in {OUTCOME_BLOCKED, OUTCOME_FAILED}:
|
|
1483
|
+
return outcome
|
|
1484
|
+
if complete:
|
|
1485
|
+
return OUTCOME_PARTIAL
|
|
1486
|
+
return OUTCOME_FAILED
|
|
1487
|
+
|
|
1488
|
+
def _cloud_text_allowed(self, bundle: ExtractionBundleV2) -> bool:
|
|
1489
|
+
"""Whether this run may use cloud text generation (opt-in, per platform)."""
|
|
1490
|
+
try:
|
|
1491
|
+
platform = bundle.sources[0].platform if bundle.sources else "youtube"
|
|
1492
|
+
return bool(self.settings.cloud_allowed(platform, "text"))
|
|
1493
|
+
except Exception: # pragma: no cover - defensive
|
|
1494
|
+
return False
|
|
1495
|
+
|
|
1496
|
+
# -------------------------------------------------------------- persist
|
|
1497
|
+
def _write_manifest(
|
|
1498
|
+
self,
|
|
1499
|
+
bundle: ExtractionBundleV2,
|
|
1500
|
+
output_dir: Path,
|
|
1501
|
+
*,
|
|
1502
|
+
requested: set[str],
|
|
1503
|
+
written: Mapping[str, Path],
|
|
1504
|
+
outcome: Outcome,
|
|
1505
|
+
format_status: Mapping[str, str],
|
|
1506
|
+
format_reasons: Mapping[str, str],
|
|
1507
|
+
reasons: Sequence[str],
|
|
1508
|
+
refresh: bool,
|
|
1509
|
+
previous_manifest: Any | None = None,
|
|
1510
|
+
doc_language: str | None = None,
|
|
1511
|
+
acquisition_mode: str = "platform",
|
|
1512
|
+
) -> Path | None:
|
|
1513
|
+
completeness = bundle.contents[0].completeness.status if bundle.contents else "unknown"
|
|
1514
|
+
acquire_stage = {"complete": "complete", "partial": "partial"}.get(completeness, "unknown")
|
|
1515
|
+
renderable = [fmt for fmt in requested if format_status.get(fmt) != "not_applicable"]
|
|
1516
|
+
render_ok = [fmt for fmt in renderable if format_status.get(fmt) == "complete"]
|
|
1517
|
+
if not renderable:
|
|
1518
|
+
render_stage = "complete"
|
|
1519
|
+
elif len(render_ok) == len(renderable):
|
|
1520
|
+
render_stage = "complete"
|
|
1521
|
+
elif render_ok:
|
|
1522
|
+
render_stage = "partial"
|
|
1523
|
+
else:
|
|
1524
|
+
render_stage = "failed"
|
|
1525
|
+
source_revision = bundle.contents[0].source_revision if bundle.contents else None
|
|
1526
|
+
manifest: dict[str, Any] = {
|
|
1527
|
+
"schema_version": 2,
|
|
1528
|
+
"source_ids": [source.source_id for source in bundle.sources],
|
|
1529
|
+
"source_revision": source_revision,
|
|
1530
|
+
"request_fingerprint": _request_fingerprint(bundle, requested),
|
|
1531
|
+
"stages": {
|
|
1532
|
+
"resolve": "complete",
|
|
1533
|
+
"acquire": acquire_stage,
|
|
1534
|
+
"segment": "complete",
|
|
1535
|
+
"extract": "complete",
|
|
1536
|
+
"render": render_stage,
|
|
1537
|
+
},
|
|
1538
|
+
"outputs": {
|
|
1539
|
+
fmt: _output_entry(
|
|
1540
|
+
fmt,
|
|
1541
|
+
status=format_status.get(fmt, "pending"),
|
|
1542
|
+
path=written.get(fmt),
|
|
1543
|
+
output_dir=output_dir,
|
|
1544
|
+
source_revision=source_revision,
|
|
1545
|
+
reason=format_reasons.get(fmt),
|
|
1546
|
+
)
|
|
1547
|
+
for fmt in sorted(requested)
|
|
1548
|
+
},
|
|
1549
|
+
"outcome": outcome,
|
|
1550
|
+
"reasons": list(reasons),
|
|
1551
|
+
"refresh": bool(refresh),
|
|
1552
|
+
"remote_tasks": dict(getattr(previous_manifest, "remote_tasks", {}) or {}),
|
|
1553
|
+
"refresh_epoch": int(getattr(previous_manifest, "refresh_epoch", 0) or 0) + (1 if refresh else 0),
|
|
1554
|
+
"updated_at": _utc_now(),
|
|
1555
|
+
}
|
|
1556
|
+
from ytk.pipeline.state import (
|
|
1557
|
+
ArtifactStoreError,
|
|
1558
|
+
ManifestV2,
|
|
1559
|
+
mark_stage,
|
|
1560
|
+
new_manifest,
|
|
1561
|
+
record_output,
|
|
1562
|
+
)
|
|
1563
|
+
|
|
1564
|
+
manifest_model = new_manifest(
|
|
1565
|
+
source_ids=[source.source_id for source in bundle.sources],
|
|
1566
|
+
request_fingerprint=manifest["request_fingerprint"],
|
|
1567
|
+
source_revision=manifest.get("source_revision"),
|
|
1568
|
+
refresh_epoch=manifest.get("refresh_epoch", 0),
|
|
1569
|
+
)
|
|
1570
|
+
for stage_name, stage_status in manifest.get("stages", {}).items():
|
|
1571
|
+
manifest_model = mark_stage(manifest_model, stage_name, stage_status)
|
|
1572
|
+
entries = {
|
|
1573
|
+
fmt: _output_entry(
|
|
1574
|
+
fmt,
|
|
1575
|
+
status=format_status.get(fmt, "pending"),
|
|
1576
|
+
path=written.get(fmt),
|
|
1577
|
+
output_dir=output_dir,
|
|
1578
|
+
source_revision=source_revision,
|
|
1579
|
+
reason=format_reasons.get(fmt),
|
|
1580
|
+
)
|
|
1581
|
+
for fmt in sorted(requested)
|
|
1582
|
+
}
|
|
1583
|
+
for fmt, entry in entries.items():
|
|
1584
|
+
manifest_model = record_output(
|
|
1585
|
+
manifest_model,
|
|
1586
|
+
fmt,
|
|
1587
|
+
status=entry["status"],
|
|
1588
|
+
path=entry["path"],
|
|
1589
|
+
sha256=entry["sha256"],
|
|
1590
|
+
validator_version=entry["validator_version"],
|
|
1591
|
+
checker_version=entry.get("checker_version"),
|
|
1592
|
+
reason=entry.get("reason"),
|
|
1593
|
+
reason_code=entry.get("reason_code"),
|
|
1594
|
+
evidence_fingerprint=entry.get("evidence_fingerprint"),
|
|
1595
|
+
)
|
|
1596
|
+
for fmt, entry_model in manifest_model.outputs.items():
|
|
1597
|
+
extra = entries.get(fmt, {})
|
|
1598
|
+
for key in ("kind", "file_count", "tree_sha256"):
|
|
1599
|
+
if key in extra:
|
|
1600
|
+
if entry_model.__pydantic_extra__ is None:
|
|
1601
|
+
entry_model.__pydantic_extra__ = {}
|
|
1602
|
+
entry_model.__pydantic_extra__[key] = extra[key]
|
|
1603
|
+
payload = manifest_model.model_dump()
|
|
1604
|
+
current_manifest = self._read_previous_manifest(output_dir)
|
|
1605
|
+
if outcome not in {OUTCOME_COMPLETE, OUTCOME_NO_KNOWLEDGE}:
|
|
1606
|
+
checkpoint = getattr(current_manifest, "acquisition_checkpoint", None)
|
|
1607
|
+
if isinstance(checkpoint, dict):
|
|
1608
|
+
payload["acquisition_checkpoint"] = checkpoint
|
|
1609
|
+
payload.update(
|
|
1610
|
+
{
|
|
1611
|
+
"outcome": outcome,
|
|
1612
|
+
"reasons": list(reasons),
|
|
1613
|
+
"refresh": bool(refresh),
|
|
1614
|
+
"remote_tasks": dict(getattr(current_manifest, "remote_tasks", {}) or manifest.get("remote_tasks", {})),
|
|
1615
|
+
"render_fingerprints": {
|
|
1616
|
+
fmt: render_request_fingerprint(fmt, self.settings, doc_language, bundle.sources[0].platform)
|
|
1617
|
+
for fmt in requested
|
|
1618
|
+
},
|
|
1619
|
+
"content_cache": {
|
|
1620
|
+
"origin": acquisition_mode,
|
|
1621
|
+
"processing_fingerprint": self._processing_fingerprint(),
|
|
1622
|
+
**{
|
|
1623
|
+
filename: _sha256_file(output_dir / filename)
|
|
1624
|
+
for filename in ("knowledge.json", "content.json", "source.json")
|
|
1625
|
+
},
|
|
1626
|
+
},
|
|
1627
|
+
"updated_at": _utc_now(),
|
|
1628
|
+
}
|
|
1629
|
+
)
|
|
1630
|
+
manifest_model = ManifestV2.model_validate(payload)
|
|
1631
|
+
store = self._store(output_dir)
|
|
1632
|
+
if store is not None:
|
|
1633
|
+
try:
|
|
1634
|
+
# write_manifest validates every claimed-complete output, asserts
|
|
1635
|
+
# ownership and writes LAST — the fail-closed layer stays active.
|
|
1636
|
+
store.write_manifest(manifest_model)
|
|
1637
|
+
return output_dir / "manifest.json"
|
|
1638
|
+
except ArtifactStoreError:
|
|
1639
|
+
raise
|
|
1640
|
+
except Exception as exc: # never lose the run over manifest plumbing
|
|
1641
|
+
LOGGER.warning("artifact store manifest write failed: %s", exc)
|
|
1642
|
+
path = output_dir / "manifest.json"
|
|
1643
|
+
_atomic_write_json(path, payload)
|
|
1644
|
+
return path
|
|
1645
|
+
|
|
1646
|
+
# -------------------------------------------------------------- helpers
|
|
1647
|
+
@staticmethod
|
|
1648
|
+
def _load_fixture_content(path: Path) -> FixtureContent:
|
|
1649
|
+
if path.is_dir():
|
|
1650
|
+
candidate = path / "content.json"
|
|
1651
|
+
if not candidate.is_file():
|
|
1652
|
+
raise UserFacingError(f"fixture directory {path} has no content.json")
|
|
1653
|
+
path = candidate
|
|
1654
|
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
1655
|
+
return FixtureContent.model_validate(payload)
|
|
1656
|
+
|
|
1657
|
+
|
|
1658
|
+
def _split_runner_output(output: Any) -> tuple[list[ContentBlock], dict[str, object]]:
|
|
1659
|
+
"""Accept either ``list[ContentBlock]`` or ``(blocks, metadata)`` from a runner."""
|
|
1660
|
+
if isinstance(output, tuple) and len(output) == 2:
|
|
1661
|
+
blocks, metadata = output
|
|
1662
|
+
return list(blocks or []), dict(metadata or {})
|
|
1663
|
+
return list(output or []), {}
|
|
1664
|
+
|
|
1665
|
+
|
|
1666
|
+
def _no_knowledge_gate(bundle: ExtractionBundleV2, completeness: Any) -> bool:
|
|
1667
|
+
"""NO_KNOWLEDGE requires the formal gate when the content model allows it."""
|
|
1668
|
+
try:
|
|
1669
|
+
from ytk.quality.completeness import decide_no_knowledge
|
|
1670
|
+
|
|
1671
|
+
if not bundle.contents:
|
|
1672
|
+
return completeness is not None and completeness.status == "complete"
|
|
1673
|
+
return decide_no_knowledge(bundle.contents[0], bundle.knowledge)
|
|
1674
|
+
except Exception: # pragma: no cover - defensive: fall back to strict check
|
|
1675
|
+
return (
|
|
1676
|
+
completeness is not None
|
|
1677
|
+
and completeness.status == "complete"
|
|
1678
|
+
and completeness.reason_code is None
|
|
1679
|
+
)
|
|
1680
|
+
|
|
1681
|
+
|
|
1682
|
+
def _default_filename(fmt: str) -> str:
|
|
1683
|
+
return {
|
|
1684
|
+
"json": "knowledge.json",
|
|
1685
|
+
"jsonl": "knowledge.jsonl",
|
|
1686
|
+
"md": "notes.md",
|
|
1687
|
+
"markdown": "notes.md",
|
|
1688
|
+
"obsidian": "obsidian.md",
|
|
1689
|
+
"summary": "summary.md",
|
|
1690
|
+
"doc": "knowledge.md",
|
|
1691
|
+
"skill": "skill",
|
|
1692
|
+
}.get(fmt, f"{fmt}.out")
|
|
1693
|
+
|
|
1694
|
+
|
|
1695
|
+
#: Bumped whenever a per-format artifact schema/validator changes. Resume must
|
|
1696
|
+
#: re-validate artifacts written by an older validator version (plan §4.5).
|
|
1697
|
+
FORMAT_VALIDATOR_VERSION = "v2-1"
|
|
1698
|
+
|
|
1699
|
+
|
|
1700
|
+
def _output_entry(
|
|
1701
|
+
fmt: str,
|
|
1702
|
+
*,
|
|
1703
|
+
status: str,
|
|
1704
|
+
path: Path | None,
|
|
1705
|
+
output_dir: Path,
|
|
1706
|
+
source_revision: str | None,
|
|
1707
|
+
reason: str | None = None,
|
|
1708
|
+
) -> dict[str, Any]:
|
|
1709
|
+
is_directory = path is not None and path.is_dir()
|
|
1710
|
+
entry: dict[str, object] = {
|
|
1711
|
+
"status": status,
|
|
1712
|
+
"path": str(path.relative_to(output_dir)) if path is not None else None,
|
|
1713
|
+
# A skill package is a directory artifact; it has no single-file digest,
|
|
1714
|
+
# so it records its kind and a deterministic file-count instead.
|
|
1715
|
+
"kind": "directory" if is_directory else ("file" if path is not None else None),
|
|
1716
|
+
"sha256": _sha256_file(path) if path is not None and not is_directory else None,
|
|
1717
|
+
"validator_version": FORMAT_VALIDATOR_VERSION,
|
|
1718
|
+
}
|
|
1719
|
+
if is_directory and path is not None:
|
|
1720
|
+
from ytk.pipeline.state import directory_digest
|
|
1721
|
+
|
|
1722
|
+
entry["file_count"] = sum(1 for item in path.rglob("*") if item.is_file())
|
|
1723
|
+
entry["tree_sha256"] = directory_digest(path)
|
|
1724
|
+
if status == "not_applicable":
|
|
1725
|
+
# An inapplicable format still records WHY and against which evidence and
|
|
1726
|
+
# checker version, so a later resume re-judges only when those change.
|
|
1727
|
+
entry["reason"] = reason or "OUTPUTS_NOT_APPLICABLE"
|
|
1728
|
+
entry["reason_code"] = reason or "OUTPUTS_NOT_APPLICABLE"
|
|
1729
|
+
entry["evidence_fingerprint"] = source_revision
|
|
1730
|
+
entry["checker_version"] = FORMAT_VALIDATOR_VERSION
|
|
1731
|
+
elif reason:
|
|
1732
|
+
entry["reason"] = reason
|
|
1733
|
+
entry["reason_code"] = reason
|
|
1734
|
+
return entry
|
|
1735
|
+
|
|
1736
|
+
|
|
1737
|
+
def render_request_fingerprint(
|
|
1738
|
+
fmt: str, settings: AppSettings, doc_language: str | None, platform: str
|
|
1739
|
+
) -> str:
|
|
1740
|
+
"""Rendering options invalidate outputs without invalidating recognition."""
|
|
1741
|
+
from ytk.renderers.knowledge_doc_renderer import detect_doc_language
|
|
1742
|
+
|
|
1743
|
+
payload: dict[str, Any] = {
|
|
1744
|
+
"format": fmt, "validator": FORMAT_VALIDATOR_VERSION,
|
|
1745
|
+
"language": detect_doc_language(doc_language or settings.doc_language),
|
|
1746
|
+
"extraction_implementation": extraction_implementation_fingerprint(),
|
|
1747
|
+
}
|
|
1748
|
+
if fmt == "doc":
|
|
1749
|
+
payload.update({
|
|
1750
|
+
"model": settings.minimax_model, "region": settings.minimax_region,
|
|
1751
|
+
"max_tokens": settings.knowledge_doc_max_tokens,
|
|
1752
|
+
"cloud_allowed": settings.cloud_allowed(cast(Any, platform), "text"),
|
|
1753
|
+
"renderer": _sha256_file(Path(__file__).parents[1] / "renderers" / "knowledge_doc_renderer.py"),
|
|
1754
|
+
})
|
|
1755
|
+
return "sha256:" + hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
|
|
1756
|
+
|
|
1757
|
+
|
|
1758
|
+
def _request_fingerprint(bundle: ExtractionBundleV2, requested: set[str]) -> str:
|
|
1759
|
+
payload = json.dumps(
|
|
1760
|
+
{
|
|
1761
|
+
"formats": sorted(requested),
|
|
1762
|
+
"revision": [content.source_revision for content in bundle.contents],
|
|
1763
|
+
"blocks": [len(content.blocks) for content in bundle.contents],
|
|
1764
|
+
},
|
|
1765
|
+
sort_keys=True,
|
|
1766
|
+
)
|
|
1767
|
+
return "sha256:" + hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]
|
|
1768
|
+
|
|
1769
|
+
|
|
1770
|
+
def _utc_now() -> str:
|
|
1771
|
+
from datetime import datetime
|
|
1772
|
+
|
|
1773
|
+
return datetime.now(UTC).isoformat(timespec="seconds")
|
|
1774
|
+
|
|
1775
|
+
|
|
1776
|
+
def _atomic_write_json(path: Path, payload: object) -> None:
|
|
1777
|
+
fd, tmp_name = tempfile.mkstemp(prefix=path.name + ".", suffix=".tmp", dir=path.parent)
|
|
1778
|
+
try:
|
|
1779
|
+
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
|
1780
|
+
handle.write(json.dumps(payload, indent=2, ensure_ascii=False) + "\n")
|
|
1781
|
+
os.replace(tmp_name, path)
|
|
1782
|
+
except BaseException:
|
|
1783
|
+
try:
|
|
1784
|
+
os.unlink(tmp_name)
|
|
1785
|
+
except OSError:
|
|
1786
|
+
pass
|
|
1787
|
+
raise
|
|
1788
|
+
|
|
1789
|
+
|
|
1790
|
+
def _copy_current_asset(source: Path, target: Path) -> None:
|
|
1791
|
+
"""Replace stale media atomically while retaining identical cached material."""
|
|
1792
|
+
if target.is_symlink() or target.parent.is_symlink() or not target.parent.resolve().is_relative_to(target.parent.parent.resolve()):
|
|
1793
|
+
raise UserFacingError("media destination uses a symbolic link or points outside the task package")
|
|
1794
|
+
if target.is_file() and _sha256_file(source) == _sha256_file(target):
|
|
1795
|
+
return
|
|
1796
|
+
fd, temporary = tempfile.mkstemp(prefix=target.name + ".", suffix=".tmp", dir=target.parent)
|
|
1797
|
+
os.close(fd)
|
|
1798
|
+
try:
|
|
1799
|
+
shutil.copyfile(source, temporary)
|
|
1800
|
+
os.replace(temporary, target)
|
|
1801
|
+
finally:
|
|
1802
|
+
Path(temporary).unlink(missing_ok=True)
|
|
1803
|
+
|
|
1804
|
+
|
|
1805
|
+
def build_v2_pipeline(
|
|
1806
|
+
settings: AppSettings | None = None,
|
|
1807
|
+
*,
|
|
1808
|
+
registry: PlatformRegistry | None = None,
|
|
1809
|
+
access: AccessContext | None = None,
|
|
1810
|
+
renderers: Mapping[str, Callable[..., None]] | None = None,
|
|
1811
|
+
coarse_atoms: bool = False,
|
|
1812
|
+
) -> MultiPlatformPipeline:
|
|
1813
|
+
"""Factory used by the CLI; wires the built-in adapters when available."""
|
|
1814
|
+
settings = settings or AppSettings()
|
|
1815
|
+
adapters = _builtin_adapters(settings)
|
|
1816
|
+
renderer_map = dict(renderers or {})
|
|
1817
|
+
if not renderer_map:
|
|
1818
|
+
try:
|
|
1819
|
+
from ytk.renderers.v2 import build_v2_renderers
|
|
1820
|
+
|
|
1821
|
+
renderer_map = build_v2_renderers()
|
|
1822
|
+
except Exception as exc: # renderers not built yet / optional dep missing
|
|
1823
|
+
LOGGER.debug("v2 renderers unavailable: %s", exc)
|
|
1824
|
+
from ytk.pipeline.runners import make_artifact_store_factory, make_asr_runner, make_ocr_runner
|
|
1825
|
+
|
|
1826
|
+
return MultiPlatformPipeline(
|
|
1827
|
+
settings=settings,
|
|
1828
|
+
registry=registry or PlatformRegistry.default(adapters=adapters),
|
|
1829
|
+
access=access,
|
|
1830
|
+
renderers=renderer_map,
|
|
1831
|
+
coarse_atoms=coarse_atoms,
|
|
1832
|
+
asr_runner=make_asr_runner(settings),
|
|
1833
|
+
ocr_runner=make_ocr_runner(settings) if settings.ocr_enabled else None,
|
|
1834
|
+
artifact_store_factory=make_artifact_store_factory(),
|
|
1835
|
+
)
|
|
1836
|
+
|
|
1837
|
+
|
|
1838
|
+
def _builtin_adapters(settings: AppSettings | None) -> dict[str, Any]:
|
|
1839
|
+
"""Instantiate the adapters that exist in this build (lazy, optional deps)."""
|
|
1840
|
+
adapters: dict[str, Any] = {}
|
|
1841
|
+
for module_name, class_name, platform in (
|
|
1842
|
+
("ytk.ingestion.adapters.youtube", "YouTubeAdapter", "youtube"),
|
|
1843
|
+
("ytk.ingestion.adapters.tiktok", "TikTokAdapter", "tiktok"),
|
|
1844
|
+
("ytk.ingestion.adapters.bilibili", "BilibiliAdapter", "bilibili"),
|
|
1845
|
+
("ytk.ingestion.adapters.douyin", "DouyinAdapter", "douyin"),
|
|
1846
|
+
("ytk.ingestion.adapters.xiaohongshu", "XiaohongshuAdapter", "xiaohongshu"),
|
|
1847
|
+
):
|
|
1848
|
+
try:
|
|
1849
|
+
module = __import__(module_name, fromlist=[class_name])
|
|
1850
|
+
adapter_class = getattr(module, class_name)
|
|
1851
|
+
adapters[platform] = adapter_class(settings=settings)
|
|
1852
|
+
except Exception as exc: # missing optional dep / not implemented yet
|
|
1853
|
+
LOGGER.debug("adapter %s unavailable: %s", platform, exc)
|
|
1854
|
+
return adapters
|
|
1855
|
+
|
|
1856
|
+
|
|
1857
|
+
__all__ = [
|
|
1858
|
+
"FixtureContent",
|
|
1859
|
+
"MultiPlatformPipeline",
|
|
1860
|
+
"Outcome",
|
|
1861
|
+
"V2RunResult",
|
|
1862
|
+
"build_v2_pipeline",
|
|
1863
|
+
]
|