kd-engine 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kd_engine-1.0.0.dist-info/METADATA +19 -0
- kd_engine-1.0.0.dist-info/RECORD +118 -0
- kd_engine-1.0.0.dist-info/WHEEL +5 -0
- kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
- kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
- kd_engine-1.0.0.dist-info/top_level.txt +1 -0
- ytk/__init__.py +2 -0
- ytk/__main__.py +21 -0
- ytk/batch/__init__.py +39 -0
- ytk/batch/runner.py +1149 -0
- ytk/cache/__init__.py +34 -0
- ytk/cache/v2.py +562 -0
- ytk/cleaning/density.py +40 -0
- ytk/cleaning/fillers.py +18 -0
- ytk/cleaning/sentence_boundary.py +30 -0
- ytk/cleaning/service.py +62 -0
- ytk/cli.py +1281 -0
- ytk/commands/__init__.py +3 -0
- ytk/commands/merge.py +600 -0
- ytk/config.py +757 -0
- ytk/dedup/service.py +166 -0
- ytk/doctor.py +891 -0
- ytk/dotenv.py +334 -0
- ytk/errors.py +38 -0
- ytk/extraction/service.py +451 -0
- ytk/extraction/strategies/base.py +16 -0
- ytk/extraction/strategies/code_block.py +285 -0
- ytk/extraction/strategies/heuristic.py +791 -0
- ytk/extraction/strategies/local_model.py +56 -0
- ytk/extraction/text_context.py +177 -0
- ytk/extraction/visual_context.py +159 -0
- ytk/ingestion/adapters/__init__.py +1 -0
- ytk/ingestion/adapters/_common.py +865 -0
- ytk/ingestion/adapters/base.py +160 -0
- ytk/ingestion/adapters/bilibili.py +271 -0
- ytk/ingestion/adapters/douyin.py +445 -0
- ytk/ingestion/adapters/tiktok.py +102 -0
- ytk/ingestion/adapters/xiaohongshu.py +758 -0
- ytk/ingestion/adapters/youtube.py +74 -0
- ytk/ingestion/browser_acquired.py +220 -0
- ytk/ingestion/http_client.py +43 -0
- ytk/ingestion/local_import.py +202 -0
- ytk/ingestion/media.py +468 -0
- ytk/ingestion/providers/asr.py +902 -0
- ytk/ingestion/providers/base.py +13 -0
- ytk/ingestion/providers/dtk_client.py +816 -0
- ytk/ingestion/providers/minimax.py +350 -0
- ytk/ingestion/providers/transcript_api.py +336 -0
- ytk/ingestion/providers/xhs_client.py +646 -0
- ytk/ingestion/providers/ytdlp.py +400 -0
- ytk/ingestion/registry.py +189 -0
- ytk/ingestion/resolver.py +417 -0
- ytk/ingestion/search.py +56 -0
- ytk/ingestion/service.py +167 -0
- ytk/ingestion/subtitles.py +602 -0
- ytk/ingestion/transcript_cache.py +212 -0
- ytk/ingestion/url_parser.py +53 -0
- ytk/ingestion/video_frames.py +292 -0
- ytk/ingestion/ytdlp_acquire.py +754 -0
- ytk/logging.py +383 -0
- ytk/models/common.py +20 -0
- ytk/models/content.py +451 -0
- ytk/models/knowledge.py +43 -0
- ytk/models/migrations.py +262 -0
- ytk/models/skill.py +13 -0
- ytk/models/source.py +265 -0
- ytk/models/topic.py +18 -0
- ytk/models/transcript.py +30 -0
- ytk/ocr/__init__.py +65 -0
- ytk/ocr/engine.py +245 -0
- ytk/ocr/layout.py +347 -0
- ytk/ocr/pipeline.py +440 -0
- ytk/ocr/rapidocr_engine.py +144 -0
- ytk/pipeline/acquisition_checkpoint.py +240 -0
- ytk/pipeline/artifacts.py +18 -0
- ytk/pipeline/orchestrator.py +598 -0
- ytk/pipeline/processing_identity.py +18 -0
- ytk/pipeline/retry.py +728 -0
- ytk/pipeline/runners.py +578 -0
- ytk/pipeline/state.py +1107 -0
- ytk/pipeline/v2.py +1863 -0
- ytk/py.typed +1 -0
- ytk/quality/__init__.py +9 -0
- ytk/quality/completeness.py +260 -0
- ytk/renderers/base.py +9 -0
- ytk/renderers/citation_alignment.py +617 -0
- ytk/renderers/evidence_formatter.py +362 -0
- ytk/renderers/json_renderer.py +13 -0
- ytk/renderers/jsonl_renderer.py +18 -0
- ytk/renderers/knowledge_doc_renderer.py +1984 -0
- ytk/renderers/markdown_renderer.py +540 -0
- ytk/renderers/obsidian_renderer.py +138 -0
- ytk/renderers/skill_package.py +21 -0
- ytk/renderers/source_links.py +309 -0
- ytk/renderers/summary_renderer.py +163 -0
- ytk/renderers/v2.py +171 -0
- ytk/renderers/v2_jsonl.py +165 -0
- ytk/renderers/v2_markdown.py +477 -0
- ytk/renderers/v2_obsidian.py +113 -0
- ytk/renderers/v2_summary.py +220 -0
- ytk/segmentation/change_points.py +50 -0
- ytk/segmentation/embeddings.py +45 -0
- ytk/segmentation/service.py +318 -0
- ytk/skill_builder/__init__.py +1 -0
- ytk/skill_builder/builder.py +419 -0
- ytk/skill_builder/templates/SKILL.md.j2 +85 -0
- ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
- ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
- ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
- ytk/skill_builder/templates/references/commands.md.j2 +34 -0
- ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
- ytk/skill_builder/templates/references/configs.md.j2 +31 -0
- ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
- ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
- ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
- ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
- ytk/skill_builder/templates.py +29 -0
- ytk/text_normalization.py +44 -0
|
@@ -0,0 +1,598 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import importlib
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import TYPE_CHECKING, Any
|
|
11
|
+
|
|
12
|
+
from pydantic import ValidationError
|
|
13
|
+
|
|
14
|
+
from ytk.cleaning.service import CleaningService
|
|
15
|
+
from ytk.config import AppSettings
|
|
16
|
+
from ytk.dedup.service import DeduplicationService
|
|
17
|
+
from ytk.errors import UserFacingError
|
|
18
|
+
from ytk.extraction.service import ExtractionService
|
|
19
|
+
from ytk.extraction.strategies.heuristic import HeuristicExtractor
|
|
20
|
+
from ytk.ingestion.providers.asr import WhisperCppAsrProvider
|
|
21
|
+
from ytk.ingestion.providers.transcript_api import TranscriptApiProvider
|
|
22
|
+
from ytk.ingestion.providers.ytdlp import YtDlpSubtitleProvider
|
|
23
|
+
from ytk.ingestion.service import IngestionService
|
|
24
|
+
from ytk.ingestion.url_parser import ParsedYouTubeUrl, parse_youtube_url
|
|
25
|
+
from ytk.models.common import TranscriptSource
|
|
26
|
+
from ytk.models.knowledge import ExtractionBundle, KnowledgeAtom
|
|
27
|
+
from ytk.models.skill import SkillSpec
|
|
28
|
+
from ytk.models.topic import TopicSegment
|
|
29
|
+
from ytk.models.transcript import TranscriptBundle, TranscriptSegment
|
|
30
|
+
|
|
31
|
+
if TYPE_CHECKING:
|
|
32
|
+
# Annotation-only import: the runtime import stays lazy inside
|
|
33
|
+
# _build_minimax_provider so importing the orchestrator never requires
|
|
34
|
+
# the minimax provider module (and its httpx dependency chain).
|
|
35
|
+
from ytk.ingestion.providers.minimax import MiniMaxVideoProvider
|
|
36
|
+
from ytk.pipeline.artifacts import ArtifactManifest
|
|
37
|
+
from ytk.renderers.base import Renderer
|
|
38
|
+
from ytk.renderers.json_renderer import JsonRenderer
|
|
39
|
+
from ytk.renderers.jsonl_renderer import JSONLRenderer
|
|
40
|
+
from ytk.renderers.knowledge_doc_renderer import KnowledgeDocRenderer, complete_doc_status
|
|
41
|
+
from ytk.renderers.markdown_renderer import MarkdownRenderer
|
|
42
|
+
from ytk.renderers.obsidian_renderer import ObsidianRenderer
|
|
43
|
+
from ytk.renderers.skill_package import SkillPackageRenderer
|
|
44
|
+
from ytk.renderers.summary_renderer import SummaryRenderer
|
|
45
|
+
from ytk.segmentation.embeddings import SentenceTransformerClient
|
|
46
|
+
from ytk.segmentation.service import TopicSegmentationService
|
|
47
|
+
from ytk.skill_builder.builder import SkillBuilder
|
|
48
|
+
|
|
49
|
+
LOGGER = logging.getLogger(__name__)
|
|
50
|
+
|
|
51
|
+
# Upper bound for ingestion retry attempts (review H2): bounds worst-case backoff
|
|
52
|
+
# stall. 10 attempts = 9 exponential backoffs/provider (1.5,3,6,12,24, then capped
|
|
53
|
+
# at 30s → ~167s/provider) × 4 providers ≈ 11 min worst case — below the cron budget.
|
|
54
|
+
_MAX_INGESTION_ATTEMPTS_CAP = 10
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _env_int(name: str, default: int) -> int:
|
|
58
|
+
"""Parse an int env var, falling back to `default` on empty/malformed values.
|
|
59
|
+
|
|
60
|
+
Review H1: `int(os.getenv(name, "3"))` crashed on a set-but-empty var
|
|
61
|
+
(`export X=`) or a typo. Fail soft to the default + log, never crash the
|
|
62
|
+
pipeline factory over a bad env value.
|
|
63
|
+
"""
|
|
64
|
+
raw = os.getenv(name)
|
|
65
|
+
try:
|
|
66
|
+
return int(raw) if raw not in (None, "") else default
|
|
67
|
+
except (TypeError, ValueError):
|
|
68
|
+
LOGGER.warning("env %s=%r is not a valid int; falling back to %d", name, raw, default)
|
|
69
|
+
return default
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _env_float(name: str, default: float) -> float:
|
|
73
|
+
raw = os.getenv(name)
|
|
74
|
+
try:
|
|
75
|
+
return float(raw) if raw not in (None, "") else default
|
|
76
|
+
except (TypeError, ValueError):
|
|
77
|
+
LOGGER.warning("env %s=%r is not a valid float; falling back to %g", name, raw, default)
|
|
78
|
+
return default
|
|
79
|
+
|
|
80
|
+
SupportedTranscriptSource = TranscriptSource
|
|
81
|
+
DEFAULT_TARGET_AGENTS = ["codex", "claude-code", "openclaw", "opencode"]
|
|
82
|
+
LOGGER = logging.getLogger(__name__)
|
|
83
|
+
DEFAULT_WHISPER_CPP_BINARY = "/usr/local/bin/whisper-cli"
|
|
84
|
+
DEFAULT_WHISPER_CPP_MODEL = str(Path.home() / ".openclaw/models/whisper/ggml-base.bin")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class DeterministicEmbeddingClient:
|
|
88
|
+
def __init__(self, dimensions: int = 12) -> None:
|
|
89
|
+
self._dimensions = dimensions
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def model_name(self) -> str:
|
|
93
|
+
return "deterministic-local"
|
|
94
|
+
|
|
95
|
+
def encode(self, texts: Sequence[str]) -> list[list[float]]:
|
|
96
|
+
return [self._encode_text(text) for text in texts]
|
|
97
|
+
|
|
98
|
+
def _encode_text(self, text: str) -> list[float]:
|
|
99
|
+
vector = [0.0] * self._dimensions
|
|
100
|
+
words = re.findall(r"[a-z0-9]+", text.casefold())
|
|
101
|
+
if not words:
|
|
102
|
+
return vector
|
|
103
|
+
|
|
104
|
+
for word in words:
|
|
105
|
+
index = sum(ord(char) for char in word) % self._dimensions
|
|
106
|
+
vector[index] += 1.0
|
|
107
|
+
return vector
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class FixtureAwareIngestionFacade:
|
|
111
|
+
def __init__(self, settings: AppSettings | None = None) -> None:
|
|
112
|
+
self._settings = settings or AppSettings()
|
|
113
|
+
self._live_ingestion: IngestionService | None = None
|
|
114
|
+
from ytk.ingestion.transcript_cache import TranscriptCache
|
|
115
|
+
|
|
116
|
+
self.transcript_cache = TranscriptCache(
|
|
117
|
+
cache_dir=self._settings.transcript_cache_dir,
|
|
118
|
+
enabled=self._settings.transcript_cache_enabled,
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
def fetch_from_url(
|
|
122
|
+
self,
|
|
123
|
+
url: str,
|
|
124
|
+
*,
|
|
125
|
+
fixture_transcript: str | Path | None = None,
|
|
126
|
+
) -> TranscriptBundle:
|
|
127
|
+
parsed_url = parse_youtube_url(url)
|
|
128
|
+
if fixture_transcript is not None:
|
|
129
|
+
# Fixtures are already local — caching them would only shadow the
|
|
130
|
+
# fixture source with a stale copy.
|
|
131
|
+
return self._fixture_bundle(parsed_url, Path(fixture_transcript))
|
|
132
|
+
languages = tuple(self._settings.transcript_languages or ("en",))
|
|
133
|
+
cached = self.transcript_cache.get(parsed_url.video_id, languages)
|
|
134
|
+
if cached is not None:
|
|
135
|
+
return cached
|
|
136
|
+
bundle = self._live_ingestion_service().fetch(parsed_url)
|
|
137
|
+
self.transcript_cache.put(bundle, languages)
|
|
138
|
+
return bundle
|
|
139
|
+
|
|
140
|
+
def _fixture_bundle(self, parsed_url: ParsedYouTubeUrl, fixture_path: Path) -> TranscriptBundle:
|
|
141
|
+
try:
|
|
142
|
+
payload = json.loads(fixture_path.read_text(encoding="utf-8"))
|
|
143
|
+
except FileNotFoundError as exc:
|
|
144
|
+
raise UserFacingError(f"Fixture transcript not found: {fixture_path}") from exc
|
|
145
|
+
except json.JSONDecodeError as exc:
|
|
146
|
+
raise UserFacingError(f"Fixture transcript is not valid JSON: {fixture_path}") from exc
|
|
147
|
+
|
|
148
|
+
transcript_source = self._infer_transcript_source(fixture_path)
|
|
149
|
+
metadata = self._base_metadata(parsed_url)
|
|
150
|
+
metadata.update(
|
|
151
|
+
{
|
|
152
|
+
"title": metadata.get("title") or f"Fixture transcript for {parsed_url.video_id}",
|
|
153
|
+
"provider_name": "fixture",
|
|
154
|
+
}
|
|
155
|
+
)
|
|
156
|
+
segments = [
|
|
157
|
+
TranscriptSegment(
|
|
158
|
+
text=str(item["text"]).strip(),
|
|
159
|
+
raw_text=str(item["text"]).strip(),
|
|
160
|
+
cleaned_text=None,
|
|
161
|
+
start=float(item["start"]),
|
|
162
|
+
end=self._segment_end(item),
|
|
163
|
+
source=transcript_source,
|
|
164
|
+
)
|
|
165
|
+
for item in payload
|
|
166
|
+
]
|
|
167
|
+
return TranscriptBundle(
|
|
168
|
+
video_id=parsed_url.video_id,
|
|
169
|
+
canonical_url=parsed_url.canonical_url,
|
|
170
|
+
transcript_source=transcript_source,
|
|
171
|
+
segments=segments,
|
|
172
|
+
metadata=metadata,
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
def _live_ingestion_service(self) -> IngestionService:
|
|
176
|
+
if self._live_ingestion is None:
|
|
177
|
+
metadata_fetcher = self._build_production_metadata_fetcher()
|
|
178
|
+
transcript_proxy = TranscriptApiProvider.build_proxy_config(self._settings)
|
|
179
|
+
transcript_languages = self._settings.transcript_languages or ("en",)
|
|
180
|
+
self._live_ingestion = IngestionService(
|
|
181
|
+
caption_providers=[
|
|
182
|
+
TranscriptApiProvider(
|
|
183
|
+
metadata_fetcher=metadata_fetcher,
|
|
184
|
+
languages=transcript_languages,
|
|
185
|
+
proxy_config=transcript_proxy,
|
|
186
|
+
),
|
|
187
|
+
],
|
|
188
|
+
subtitle_fallback_provider=YtDlpSubtitleProvider(
|
|
189
|
+
metadata_fetcher=metadata_fetcher,
|
|
190
|
+
language=transcript_languages[0],
|
|
191
|
+
fallback_languages=tuple(transcript_languages[1:]),
|
|
192
|
+
proxy_url=self._settings.transcript_http_proxy_url,
|
|
193
|
+
cookies_file=self._settings.yt_dlp_cookies_file,
|
|
194
|
+
cookies_from_browser=self._settings.yt_dlp_cookies_from_browser,
|
|
195
|
+
),
|
|
196
|
+
audio_fallback_provider=WhisperCppAsrProvider(
|
|
197
|
+
metadata_fetcher=metadata_fetcher,
|
|
198
|
+
whisper_cpp_binary=self._settings.whisper_cpp_binary or DEFAULT_WHISPER_CPP_BINARY,
|
|
199
|
+
model_path=self._settings.whisper_cpp_model or DEFAULT_WHISPER_CPP_MODEL,
|
|
200
|
+
language=self._settings.whisper_cpp_language,
|
|
201
|
+
ffmpeg_binary=self._settings.ffmpeg_binary,
|
|
202
|
+
yt_dlp_binary=self._settings.yt_dlp_binary,
|
|
203
|
+
cookies_file=self._settings.yt_dlp_cookies_file,
|
|
204
|
+
cookies_from_browser=self._settings.yt_dlp_cookies_from_browser,
|
|
205
|
+
),
|
|
206
|
+
minimax_fallback_provider=self._build_minimax_provider(metadata_fetcher) if self._settings.minimax_api_key else None,
|
|
207
|
+
# Retry/backoff (§4.1): production opts in here. Env-overridable so a
|
|
208
|
+
# flaky 01:00 cron can be tuned without a code change. Default 3
|
|
209
|
+
# attempts, 1.5s base (exponential ×2 + jitter, capped 30s). Safe
|
|
210
|
+
# parse + cap (review H1/H2): bad/empty env won't crash the factory
|
|
211
|
+
# and attempts can't stall the cron for hours.
|
|
212
|
+
max_attempts=min(_env_int("YTK_INGESTION_MAX_ATTEMPTS", 3), _MAX_INGESTION_ATTEMPTS_CAP),
|
|
213
|
+
base_delay=_env_float("YTK_INGESTION_BASE_DELAY", 1.5),
|
|
214
|
+
)
|
|
215
|
+
return self._live_ingestion
|
|
216
|
+
|
|
217
|
+
def _build_minimax_provider(self, metadata_fetcher: Callable) -> MiniMaxVideoProvider | None:
|
|
218
|
+
from ytk.ingestion.providers.minimax import MiniMaxVideoProvider
|
|
219
|
+
|
|
220
|
+
return MiniMaxVideoProvider(
|
|
221
|
+
metadata_fetcher=metadata_fetcher,
|
|
222
|
+
api_key=self._settings.minimax_api_key,
|
|
223
|
+
region=self._settings.minimax_region,
|
|
224
|
+
model=self._settings.minimax_model,
|
|
225
|
+
max_tokens=self._settings.minimax_max_tokens,
|
|
226
|
+
video_detail=self._settings.minimax_video_detail,
|
|
227
|
+
video_fps=self._settings.minimax_video_fps,
|
|
228
|
+
max_long_side_pixel=self._settings.minimax_max_long_side_pixel,
|
|
229
|
+
max_video_mb=self._settings.minimax_max_video_mb,
|
|
230
|
+
yt_dlp_binary=self._settings.yt_dlp_binary,
|
|
231
|
+
cookies_file=self._settings.yt_dlp_cookies_file,
|
|
232
|
+
cookies_from_browser=self._settings.yt_dlp_cookies_from_browser,
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
@staticmethod
|
|
236
|
+
def _segment_end(item: dict[str, Any]) -> float:
|
|
237
|
+
if "end" in item:
|
|
238
|
+
return float(item["end"])
|
|
239
|
+
duration = float(item.get("duration", 0.0))
|
|
240
|
+
return float(item["start"]) + duration
|
|
241
|
+
|
|
242
|
+
@staticmethod
|
|
243
|
+
def _infer_transcript_source(fixture_path: Path) -> SupportedTranscriptSource:
|
|
244
|
+
name = fixture_path.stem.casefold()
|
|
245
|
+
if "minimax" in name:
|
|
246
|
+
return "minimax"
|
|
247
|
+
if "asr" in name:
|
|
248
|
+
return "asr"
|
|
249
|
+
if "auto" in name:
|
|
250
|
+
return "auto"
|
|
251
|
+
if "yt" in name:
|
|
252
|
+
return "yt_dlp"
|
|
253
|
+
return "manual"
|
|
254
|
+
|
|
255
|
+
def _build_production_metadata_fetcher(self) -> Callable[[ParsedYouTubeUrl], Mapping[str, object]]:
|
|
256
|
+
"""Enrich the base metadata stub with yt-dlp (title/duration/chapters).
|
|
257
|
+
|
|
258
|
+
Roadmap 2026-10-06 §3.1.4 (bd zui): without this step the production
|
|
259
|
+
primary provider never saw chapters, so chapter-aligned segmentation
|
|
260
|
+
(``segmentation_source: "chapter"``) and the hour-aware citation
|
|
261
|
+
contract were dead code paths. yt-dlp is an existing dependency; the
|
|
262
|
+
metadata-only network step was approved via roadmap §8.6 + the user's
|
|
263
|
+
2026-10-06 implementation authorization. Failure degrades to the bare
|
|
264
|
+
stub inside YtDlpMetadataFetcher — transcription still proceeds.
|
|
265
|
+
"""
|
|
266
|
+
from ytk.ingestion.providers.ytdlp import YtDlpMetadataFetcher
|
|
267
|
+
|
|
268
|
+
return YtDlpMetadataFetcher(
|
|
269
|
+
fallback=self._base_metadata,
|
|
270
|
+
proxy_url=self._settings.transcript_http_proxy_url,
|
|
271
|
+
cookies_file=self._settings.yt_dlp_cookies_file,
|
|
272
|
+
cookies_from_browser=self._settings.yt_dlp_cookies_from_browser,
|
|
273
|
+
)
|
|
274
|
+
|
|
275
|
+
@staticmethod
|
|
276
|
+
def _base_metadata(parsed_url: ParsedYouTubeUrl) -> dict[str, object]:
|
|
277
|
+
return {
|
|
278
|
+
"video_id": parsed_url.video_id,
|
|
279
|
+
"canonical_url": parsed_url.canonical_url,
|
|
280
|
+
"title": f"YouTube video {parsed_url.video_id}",
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
class ExtractionPipeline:
|
|
285
|
+
def __init__(
|
|
286
|
+
self,
|
|
287
|
+
ingestion: FixtureAwareIngestionFacade,
|
|
288
|
+
cleaning: CleaningService,
|
|
289
|
+
segmentation: TopicSegmentationService,
|
|
290
|
+
extraction: ExtractionService,
|
|
291
|
+
dedup: DeduplicationService,
|
|
292
|
+
renderers: dict[str, Renderer],
|
|
293
|
+
skill_builder: SkillBuilder,
|
|
294
|
+
target_agents: Sequence[str] | None = None,
|
|
295
|
+
) -> None:
|
|
296
|
+
self.ingestion = ingestion
|
|
297
|
+
self.cleaning = cleaning
|
|
298
|
+
self.segmentation = segmentation
|
|
299
|
+
self.extraction = extraction
|
|
300
|
+
self.dedup = dedup
|
|
301
|
+
self.renderers = dict(renderers)
|
|
302
|
+
self.skill_builder = skill_builder
|
|
303
|
+
self.target_agents = list(target_agents or DEFAULT_TARGET_AGENTS)
|
|
304
|
+
|
|
305
|
+
def run(
|
|
306
|
+
self,
|
|
307
|
+
*,
|
|
308
|
+
url: str,
|
|
309
|
+
output_dir: Path,
|
|
310
|
+
formats: set[str],
|
|
311
|
+
fixture_transcript: str | Path | None = None,
|
|
312
|
+
doc_language: str | None = None,
|
|
313
|
+
with_reviewer_scaffolding: bool = False,
|
|
314
|
+
) -> ExtractionBundle:
|
|
315
|
+
manifest = self._manifest_for(output_dir)
|
|
316
|
+
transcript_bundle = self.ingestion.fetch_from_url(url, fixture_transcript=fixture_transcript)
|
|
317
|
+
if fixture_transcript is not None and manifest.metadata_json.exists():
|
|
318
|
+
# A local transcript fixture is not a source of real video metadata.
|
|
319
|
+
# Preserve existing metadata byte-for-byte, but only when its video
|
|
320
|
+
# identity is unambiguous and matches the requested video.
|
|
321
|
+
try:
|
|
322
|
+
existing = json.loads(manifest.metadata_json.read_text(encoding="utf-8"))
|
|
323
|
+
if (not isinstance(existing, dict) or existing.get("video_id") != transcript_bundle.video_id
|
|
324
|
+
or not isinstance(existing.get("title"), str) or not existing["title"].strip()):
|
|
325
|
+
raise ValueError("invalid or mismatched existing video metadata")
|
|
326
|
+
canonical = existing.get("canonical_url")
|
|
327
|
+
if canonical is not None and (not isinstance(canonical, str)
|
|
328
|
+
or parse_youtube_url(canonical).video_id != transcript_bundle.video_id):
|
|
329
|
+
raise ValueError("existing metadata URL does not match video")
|
|
330
|
+
except (OSError, UnicodeError, ValueError, TypeError, UserFacingError) as exc:
|
|
331
|
+
raise RuntimeError("fixture cannot replace invalid or mismatched existing metadata") from exc
|
|
332
|
+
transcript_bundle = transcript_bundle.model_copy(update={"metadata": existing})
|
|
333
|
+
else:
|
|
334
|
+
self._persist_stage_json(manifest.metadata_json, transcript_bundle.metadata)
|
|
335
|
+
self._persist_stage_json(manifest.raw_transcript_json, transcript_bundle)
|
|
336
|
+
|
|
337
|
+
cleaned = self.cleaning.clean(transcript_bundle.segments)
|
|
338
|
+
self._persist_stage_json(manifest.cleaned_transcript_json, cleaned)
|
|
339
|
+
|
|
340
|
+
raw_chapters = transcript_bundle.metadata.get("chapters")
|
|
341
|
+
chapters = raw_chapters if isinstance(raw_chapters, list) else None
|
|
342
|
+
topics = self.segmentation.segment(cleaned, chapters=chapters)
|
|
343
|
+
self._persist_stage_json(manifest.topic_segments_json, topics)
|
|
344
|
+
|
|
345
|
+
atoms = self._extract_atoms(topics, cleaned, transcript_bundle.video_id)
|
|
346
|
+
merged = self.dedup.merge(atoms)
|
|
347
|
+
bundle = self._build_bundle(transcript_bundle, topics, merged)
|
|
348
|
+
self._persist_outputs(
|
|
349
|
+
bundle,
|
|
350
|
+
manifest,
|
|
351
|
+
formats,
|
|
352
|
+
doc_language=doc_language,
|
|
353
|
+
with_reviewer_scaffolding=with_reviewer_scaffolding,
|
|
354
|
+
)
|
|
355
|
+
return bundle
|
|
356
|
+
|
|
357
|
+
def _extract_atoms(
|
|
358
|
+
self,
|
|
359
|
+
topics: list[TopicSegment],
|
|
360
|
+
cleaned_segments: list[TranscriptSegment],
|
|
361
|
+
source_video: str,
|
|
362
|
+
) -> list[KnowledgeAtom]:
|
|
363
|
+
if not topics:
|
|
364
|
+
return self.extraction.extract(cleaned_segments, ["Transcript"], source_video)
|
|
365
|
+
|
|
366
|
+
atoms: list[KnowledgeAtom] = []
|
|
367
|
+
for topic in topics:
|
|
368
|
+
topic_segments = self._segments_for_topic(cleaned_segments, topic)
|
|
369
|
+
if not topic_segments:
|
|
370
|
+
continue
|
|
371
|
+
atoms.extend(
|
|
372
|
+
self.extraction.extract(
|
|
373
|
+
topic_segments,
|
|
374
|
+
topic_path=[topic.title, topic.id],
|
|
375
|
+
source_video=source_video,
|
|
376
|
+
)
|
|
377
|
+
)
|
|
378
|
+
return atoms
|
|
379
|
+
|
|
380
|
+
@staticmethod
|
|
381
|
+
def _segments_for_topic(
|
|
382
|
+
cleaned_segments: Sequence[TranscriptSegment],
|
|
383
|
+
topic: TopicSegment,
|
|
384
|
+
) -> list[TranscriptSegment]:
|
|
385
|
+
return [
|
|
386
|
+
segment
|
|
387
|
+
for segment in cleaned_segments
|
|
388
|
+
if segment.start >= topic.start and segment.start < topic.end
|
|
389
|
+
]
|
|
390
|
+
|
|
391
|
+
def _build_bundle(
|
|
392
|
+
self,
|
|
393
|
+
transcript_bundle: TranscriptBundle,
|
|
394
|
+
topics: list[TopicSegment],
|
|
395
|
+
knowledge_atoms: list[KnowledgeAtom],
|
|
396
|
+
) -> ExtractionBundle:
|
|
397
|
+
metadata = dict(transcript_bundle.metadata)
|
|
398
|
+
metadata.setdefault("canonical_url", transcript_bundle.canonical_url)
|
|
399
|
+
metadata.setdefault("title", f"YouTube video {transcript_bundle.video_id}")
|
|
400
|
+
return ExtractionBundle(
|
|
401
|
+
video_id=transcript_bundle.video_id,
|
|
402
|
+
metadata=metadata,
|
|
403
|
+
transcript_source=transcript_bundle.transcript_source,
|
|
404
|
+
topic_segments=topics,
|
|
405
|
+
knowledge_atoms=knowledge_atoms,
|
|
406
|
+
skill_spec=self._build_skill_spec(transcript_bundle, topics),
|
|
407
|
+
)
|
|
408
|
+
|
|
409
|
+
def _build_skill_spec(
|
|
410
|
+
self,
|
|
411
|
+
transcript_bundle: TranscriptBundle,
|
|
412
|
+
topics: list[TopicSegment],
|
|
413
|
+
) -> SkillSpec:
|
|
414
|
+
title = str(transcript_bundle.metadata.get("title") or transcript_bundle.video_id)
|
|
415
|
+
return SkillSpec(
|
|
416
|
+
skill_name=self._slugify(title) or f"youtube-video-{transcript_bundle.video_id}",
|
|
417
|
+
purpose="Turn a tutorial transcript into a reusable, evidence-grounded skill package.",
|
|
418
|
+
trigger_description=(
|
|
419
|
+
"Example output generated deterministically by "
|
|
420
|
+
"knowledge-distiller from this source video — a provenance "
|
|
421
|
+
"demo of the skill format, not an installable standalone skill."
|
|
422
|
+
),
|
|
423
|
+
target_agents=list(self.target_agents),
|
|
424
|
+
required_files=["SKILL.md", "references/source-summary.md"],
|
|
425
|
+
generated_files=[
|
|
426
|
+
"references/commands.md",
|
|
427
|
+
"references/configs.md",
|
|
428
|
+
"references/concepts.md",
|
|
429
|
+
"examples/example-output.md",
|
|
430
|
+
],
|
|
431
|
+
references_map={topic.id: topic.title for topic in topics},
|
|
432
|
+
)
|
|
433
|
+
|
|
434
|
+
@staticmethod
|
|
435
|
+
def _slugify(value: str) -> str:
|
|
436
|
+
slug = re.sub(r"[^a-z0-9]+", "-", value.casefold()).strip("-")
|
|
437
|
+
return slug[:80]
|
|
438
|
+
|
|
439
|
+
@staticmethod
|
|
440
|
+
def _manifest_for(output_dir: Path) -> ArtifactManifest:
|
|
441
|
+
output_dir = Path(output_dir)
|
|
442
|
+
output_dir.mkdir(parents=True, exist_ok=True)
|
|
443
|
+
return ArtifactManifest(
|
|
444
|
+
root=output_dir,
|
|
445
|
+
metadata_json=output_dir / "metadata.json",
|
|
446
|
+
raw_transcript_json=output_dir / "raw-transcript.json",
|
|
447
|
+
cleaned_transcript_json=output_dir / "cleaned-transcript.json",
|
|
448
|
+
topic_segments_json=output_dir / "topic-segments.json",
|
|
449
|
+
knowledge_json=output_dir / "knowledge.json",
|
|
450
|
+
notes_md=output_dir / "notes.md",
|
|
451
|
+
knowledge_summary_md=output_dir / "summary.md",
|
|
452
|
+
obsidian_md=output_dir / "obsidian.md",
|
|
453
|
+
skill_dir=output_dir / "skill",
|
|
454
|
+
)
|
|
455
|
+
|
|
456
|
+
@staticmethod
|
|
457
|
+
def _persist_stage_json(output_path: Path, payload: Any) -> None:
|
|
458
|
+
if hasattr(payload, "model_dump"):
|
|
459
|
+
serializable = payload.model_dump(mode="json")
|
|
460
|
+
elif isinstance(payload, list):
|
|
461
|
+
serializable = [
|
|
462
|
+
item.model_dump(mode="json") if hasattr(item, "model_dump") else item for item in payload
|
|
463
|
+
]
|
|
464
|
+
else:
|
|
465
|
+
serializable = payload
|
|
466
|
+
|
|
467
|
+
output_path.write_text(
|
|
468
|
+
json.dumps(serializable, indent=2, ensure_ascii=False) + "\n",
|
|
469
|
+
encoding="utf-8",
|
|
470
|
+
)
|
|
471
|
+
|
|
472
|
+
def _persist_outputs(
|
|
473
|
+
self,
|
|
474
|
+
bundle: ExtractionBundle,
|
|
475
|
+
manifest: ArtifactManifest,
|
|
476
|
+
formats: set[str],
|
|
477
|
+
doc_language: str | None = None,
|
|
478
|
+
with_reviewer_scaffolding: bool = False,
|
|
479
|
+
) -> None:
|
|
480
|
+
unsupported = formats - self.renderers.keys()
|
|
481
|
+
if unsupported:
|
|
482
|
+
supported = ", ".join(sorted(self.renderers))
|
|
483
|
+
unknown = ", ".join(sorted(unsupported))
|
|
484
|
+
raise ValueError(f"Unsupported format(s): {unknown}. Expected one of: {supported}")
|
|
485
|
+
|
|
486
|
+
format_to_path: dict[str, Path] = {
|
|
487
|
+
"json": manifest.knowledge_json,
|
|
488
|
+
"jsonl": manifest.root / "knowledge.jsonl",
|
|
489
|
+
"md": manifest.notes_md,
|
|
490
|
+
"obsidian": manifest.obsidian_md,
|
|
491
|
+
"summary": manifest.knowledge_summary_md,
|
|
492
|
+
"doc": manifest.root / "knowledge.md",
|
|
493
|
+
}
|
|
494
|
+
# Build per-format render kwargs (only `doc` currently accepts the
|
|
495
|
+
# reviewer-scaffolding flag; other renderers ignore it).
|
|
496
|
+
render_kwargs: dict[str, dict[str, Any]] = {
|
|
497
|
+
"doc": {"with_reviewer_scaffolding": with_reviewer_scaffolding},
|
|
498
|
+
}
|
|
499
|
+
if "skill" in formats:
|
|
500
|
+
# v1 semantics (A01): the legacy path always builds the provenance
|
|
501
|
+
# package; NOT_APPLICABLE refusal is a v2-surface behavior only.
|
|
502
|
+
self.renderers["skill"].render(
|
|
503
|
+
bundle,
|
|
504
|
+
manifest.skill_dir,
|
|
505
|
+
package_kind="generic",
|
|
506
|
+
require_actionable=False,
|
|
507
|
+
)
|
|
508
|
+
with self._doc_language_scope(doc_language):
|
|
509
|
+
# Preserve independent non-doc artifacts even if the final doc is
|
|
510
|
+
# degraded; never report the overall extraction as complete then.
|
|
511
|
+
for fmt in sorted(formats, key=lambda value: (value == "doc", value)):
|
|
512
|
+
if fmt == "skill":
|
|
513
|
+
continue # handled above with kwargs
|
|
514
|
+
if fmt in format_to_path:
|
|
515
|
+
renderer = self.renderers[fmt]
|
|
516
|
+
renderer.render(
|
|
517
|
+
bundle, format_to_path[fmt], **render_kwargs.get(fmt, {})
|
|
518
|
+
)
|
|
519
|
+
if fmt == "doc" and isinstance(renderer, KnowledgeDocRenderer):
|
|
520
|
+
if not complete_doc_status(manifest.root):
|
|
521
|
+
message = "knowledge document is degraded or unverified; see .knowledge-doc-status.json"
|
|
522
|
+
if os.environ.get("YTK_REQUIRE_COMPLETE_DOC", "").strip().lower() in {"1", "true", "yes", "on"}:
|
|
523
|
+
raise RuntimeError(message)
|
|
524
|
+
LOGGER.warning(message)
|
|
525
|
+
|
|
526
|
+
def _doc_language_scope(self, doc_language: str | None):
|
|
527
|
+
"""Context manager: temporarily swap the doc renderer for the requested language.
|
|
528
|
+
|
|
529
|
+
Exposed as a separate method (not just an inline block) so unit tests
|
|
530
|
+
can exercise the swap without standing up the full pipeline.
|
|
531
|
+
"""
|
|
532
|
+
from contextlib import contextmanager
|
|
533
|
+
|
|
534
|
+
from ytk.renderers.knowledge_doc_renderer import KnowledgeDocRenderer
|
|
535
|
+
|
|
536
|
+
@contextmanager
|
|
537
|
+
def _scope():
|
|
538
|
+
if not (doc_language and "doc" in self.renderers):
|
|
539
|
+
yield
|
|
540
|
+
return
|
|
541
|
+
original = self.renderers["doc"]
|
|
542
|
+
self.renderers["doc"] = KnowledgeDocRenderer(language=doc_language)
|
|
543
|
+
try:
|
|
544
|
+
yield
|
|
545
|
+
finally:
|
|
546
|
+
self.renderers["doc"] = original
|
|
547
|
+
|
|
548
|
+
return _scope()
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def build_default_pipeline(settings: AppSettings | None = None) -> ExtractionPipeline:
|
|
552
|
+
settings = settings or AppSettings()
|
|
553
|
+
skill_builder = SkillBuilder()
|
|
554
|
+
return ExtractionPipeline(
|
|
555
|
+
ingestion=FixtureAwareIngestionFacade(settings),
|
|
556
|
+
cleaning=CleaningService(),
|
|
557
|
+
segmentation=TopicSegmentationService(
|
|
558
|
+
embedding_client=_build_default_embedding_client(settings),
|
|
559
|
+
),
|
|
560
|
+
extraction=ExtractionService(default_strategy=HeuristicExtractor()),
|
|
561
|
+
dedup=DeduplicationService(),
|
|
562
|
+
renderers={
|
|
563
|
+
"json": JsonRenderer(),
|
|
564
|
+
"jsonl": JSONLRenderer(),
|
|
565
|
+
"md": MarkdownRenderer(),
|
|
566
|
+
"obsidian": ObsidianRenderer(),
|
|
567
|
+
"summary": SummaryRenderer(),
|
|
568
|
+
"doc": KnowledgeDocRenderer(),
|
|
569
|
+
"skill": SkillPackageRenderer(builder=skill_builder),
|
|
570
|
+
},
|
|
571
|
+
skill_builder=skill_builder,
|
|
572
|
+
)
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def _build_default_embedding_client(settings: AppSettings) -> SentenceTransformerClient | DeterministicEmbeddingClient:
|
|
576
|
+
try:
|
|
577
|
+
importlib.import_module("sentence_transformers")
|
|
578
|
+
return SentenceTransformerClient(model_name=settings.sentence_transformer_model)
|
|
579
|
+
except Exception as exc:
|
|
580
|
+
LOGGER.warning(
|
|
581
|
+
"Falling back to deterministic embeddings because sentence-transformers is unavailable: %s",
|
|
582
|
+
exc,
|
|
583
|
+
)
|
|
584
|
+
return DeterministicEmbeddingClient()
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
def load_bundle(path: Path) -> ExtractionBundle:
|
|
588
|
+
try:
|
|
589
|
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
590
|
+
except FileNotFoundError as exc:
|
|
591
|
+
raise UserFacingError(f"Knowledge bundle not found: {path}") from exc
|
|
592
|
+
except json.JSONDecodeError as exc:
|
|
593
|
+
raise UserFacingError(f"Knowledge bundle is not valid JSON: {path}") from exc
|
|
594
|
+
|
|
595
|
+
try:
|
|
596
|
+
return ExtractionBundle.model_validate(payload)
|
|
597
|
+
except ValidationError as exc:
|
|
598
|
+
raise UserFacingError(f"Failed to load knowledge bundle: {exc}") from exc
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Implementation identity for extraction judgments and their rendered output."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def extraction_implementation_fingerprint(package_root: Path | None = None) -> str:
|
|
11
|
+
root = package_root or Path(__file__).resolve().parents[1]
|
|
12
|
+
paths = [*sorted((root / "extraction").rglob("*.py")), root / "text_normalization.py"]
|
|
13
|
+
digests = {}
|
|
14
|
+
for path in paths:
|
|
15
|
+
with path.open("rb") as handle:
|
|
16
|
+
digests[path.relative_to(root).as_posix()] = hashlib.file_digest(handle, "sha256").hexdigest()
|
|
17
|
+
payload = json.dumps(digests, sort_keys=True, separators=(",", ":")).encode()
|
|
18
|
+
return "sha256:" + hashlib.sha256(payload).hexdigest()
|