kd-engine 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. kd_engine-1.0.0.dist-info/METADATA +19 -0
  2. kd_engine-1.0.0.dist-info/RECORD +118 -0
  3. kd_engine-1.0.0.dist-info/WHEEL +5 -0
  4. kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
  5. kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
  6. kd_engine-1.0.0.dist-info/top_level.txt +1 -0
  7. ytk/__init__.py +2 -0
  8. ytk/__main__.py +21 -0
  9. ytk/batch/__init__.py +39 -0
  10. ytk/batch/runner.py +1149 -0
  11. ytk/cache/__init__.py +34 -0
  12. ytk/cache/v2.py +562 -0
  13. ytk/cleaning/density.py +40 -0
  14. ytk/cleaning/fillers.py +18 -0
  15. ytk/cleaning/sentence_boundary.py +30 -0
  16. ytk/cleaning/service.py +62 -0
  17. ytk/cli.py +1281 -0
  18. ytk/commands/__init__.py +3 -0
  19. ytk/commands/merge.py +600 -0
  20. ytk/config.py +757 -0
  21. ytk/dedup/service.py +166 -0
  22. ytk/doctor.py +891 -0
  23. ytk/dotenv.py +334 -0
  24. ytk/errors.py +38 -0
  25. ytk/extraction/service.py +451 -0
  26. ytk/extraction/strategies/base.py +16 -0
  27. ytk/extraction/strategies/code_block.py +285 -0
  28. ytk/extraction/strategies/heuristic.py +791 -0
  29. ytk/extraction/strategies/local_model.py +56 -0
  30. ytk/extraction/text_context.py +177 -0
  31. ytk/extraction/visual_context.py +159 -0
  32. ytk/ingestion/adapters/__init__.py +1 -0
  33. ytk/ingestion/adapters/_common.py +865 -0
  34. ytk/ingestion/adapters/base.py +160 -0
  35. ytk/ingestion/adapters/bilibili.py +271 -0
  36. ytk/ingestion/adapters/douyin.py +445 -0
  37. ytk/ingestion/adapters/tiktok.py +102 -0
  38. ytk/ingestion/adapters/xiaohongshu.py +758 -0
  39. ytk/ingestion/adapters/youtube.py +74 -0
  40. ytk/ingestion/browser_acquired.py +220 -0
  41. ytk/ingestion/http_client.py +43 -0
  42. ytk/ingestion/local_import.py +202 -0
  43. ytk/ingestion/media.py +468 -0
  44. ytk/ingestion/providers/asr.py +902 -0
  45. ytk/ingestion/providers/base.py +13 -0
  46. ytk/ingestion/providers/dtk_client.py +816 -0
  47. ytk/ingestion/providers/minimax.py +350 -0
  48. ytk/ingestion/providers/transcript_api.py +336 -0
  49. ytk/ingestion/providers/xhs_client.py +646 -0
  50. ytk/ingestion/providers/ytdlp.py +400 -0
  51. ytk/ingestion/registry.py +189 -0
  52. ytk/ingestion/resolver.py +417 -0
  53. ytk/ingestion/search.py +56 -0
  54. ytk/ingestion/service.py +167 -0
  55. ytk/ingestion/subtitles.py +602 -0
  56. ytk/ingestion/transcript_cache.py +212 -0
  57. ytk/ingestion/url_parser.py +53 -0
  58. ytk/ingestion/video_frames.py +292 -0
  59. ytk/ingestion/ytdlp_acquire.py +754 -0
  60. ytk/logging.py +383 -0
  61. ytk/models/common.py +20 -0
  62. ytk/models/content.py +451 -0
  63. ytk/models/knowledge.py +43 -0
  64. ytk/models/migrations.py +262 -0
  65. ytk/models/skill.py +13 -0
  66. ytk/models/source.py +265 -0
  67. ytk/models/topic.py +18 -0
  68. ytk/models/transcript.py +30 -0
  69. ytk/ocr/__init__.py +65 -0
  70. ytk/ocr/engine.py +245 -0
  71. ytk/ocr/layout.py +347 -0
  72. ytk/ocr/pipeline.py +440 -0
  73. ytk/ocr/rapidocr_engine.py +144 -0
  74. ytk/pipeline/acquisition_checkpoint.py +240 -0
  75. ytk/pipeline/artifacts.py +18 -0
  76. ytk/pipeline/orchestrator.py +598 -0
  77. ytk/pipeline/processing_identity.py +18 -0
  78. ytk/pipeline/retry.py +728 -0
  79. ytk/pipeline/runners.py +578 -0
  80. ytk/pipeline/state.py +1107 -0
  81. ytk/pipeline/v2.py +1863 -0
  82. ytk/py.typed +1 -0
  83. ytk/quality/__init__.py +9 -0
  84. ytk/quality/completeness.py +260 -0
  85. ytk/renderers/base.py +9 -0
  86. ytk/renderers/citation_alignment.py +617 -0
  87. ytk/renderers/evidence_formatter.py +362 -0
  88. ytk/renderers/json_renderer.py +13 -0
  89. ytk/renderers/jsonl_renderer.py +18 -0
  90. ytk/renderers/knowledge_doc_renderer.py +1984 -0
  91. ytk/renderers/markdown_renderer.py +540 -0
  92. ytk/renderers/obsidian_renderer.py +138 -0
  93. ytk/renderers/skill_package.py +21 -0
  94. ytk/renderers/source_links.py +309 -0
  95. ytk/renderers/summary_renderer.py +163 -0
  96. ytk/renderers/v2.py +171 -0
  97. ytk/renderers/v2_jsonl.py +165 -0
  98. ytk/renderers/v2_markdown.py +477 -0
  99. ytk/renderers/v2_obsidian.py +113 -0
  100. ytk/renderers/v2_summary.py +220 -0
  101. ytk/segmentation/change_points.py +50 -0
  102. ytk/segmentation/embeddings.py +45 -0
  103. ytk/segmentation/service.py +318 -0
  104. ytk/skill_builder/__init__.py +1 -0
  105. ytk/skill_builder/builder.py +419 -0
  106. ytk/skill_builder/templates/SKILL.md.j2 +85 -0
  107. ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
  108. ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
  109. ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
  110. ytk/skill_builder/templates/references/commands.md.j2 +34 -0
  111. ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
  112. ytk/skill_builder/templates/references/configs.md.j2 +31 -0
  113. ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
  114. ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
  115. ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
  116. ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
  117. ytk/skill_builder/templates.py +29 -0
  118. ytk/text_normalization.py +44 -0
@@ -0,0 +1,598 @@
1
+ from __future__ import annotations
2
+
3
+ import importlib
4
+ import json
5
+ import logging
6
+ import os
7
+ import re
8
+ from collections.abc import Callable, Mapping, Sequence
9
+ from pathlib import Path
10
+ from typing import TYPE_CHECKING, Any
11
+
12
+ from pydantic import ValidationError
13
+
14
+ from ytk.cleaning.service import CleaningService
15
+ from ytk.config import AppSettings
16
+ from ytk.dedup.service import DeduplicationService
17
+ from ytk.errors import UserFacingError
18
+ from ytk.extraction.service import ExtractionService
19
+ from ytk.extraction.strategies.heuristic import HeuristicExtractor
20
+ from ytk.ingestion.providers.asr import WhisperCppAsrProvider
21
+ from ytk.ingestion.providers.transcript_api import TranscriptApiProvider
22
+ from ytk.ingestion.providers.ytdlp import YtDlpSubtitleProvider
23
+ from ytk.ingestion.service import IngestionService
24
+ from ytk.ingestion.url_parser import ParsedYouTubeUrl, parse_youtube_url
25
+ from ytk.models.common import TranscriptSource
26
+ from ytk.models.knowledge import ExtractionBundle, KnowledgeAtom
27
+ from ytk.models.skill import SkillSpec
28
+ from ytk.models.topic import TopicSegment
29
+ from ytk.models.transcript import TranscriptBundle, TranscriptSegment
30
+
31
+ if TYPE_CHECKING:
32
+ # Annotation-only import: the runtime import stays lazy inside
33
+ # _build_minimax_provider so importing the orchestrator never requires
34
+ # the minimax provider module (and its httpx dependency chain).
35
+ from ytk.ingestion.providers.minimax import MiniMaxVideoProvider
36
+ from ytk.pipeline.artifacts import ArtifactManifest
37
+ from ytk.renderers.base import Renderer
38
+ from ytk.renderers.json_renderer import JsonRenderer
39
+ from ytk.renderers.jsonl_renderer import JSONLRenderer
40
+ from ytk.renderers.knowledge_doc_renderer import KnowledgeDocRenderer, complete_doc_status
41
+ from ytk.renderers.markdown_renderer import MarkdownRenderer
42
+ from ytk.renderers.obsidian_renderer import ObsidianRenderer
43
+ from ytk.renderers.skill_package import SkillPackageRenderer
44
+ from ytk.renderers.summary_renderer import SummaryRenderer
45
+ from ytk.segmentation.embeddings import SentenceTransformerClient
46
+ from ytk.segmentation.service import TopicSegmentationService
47
+ from ytk.skill_builder.builder import SkillBuilder
48
+
49
+ LOGGER = logging.getLogger(__name__)
50
+
51
+ # Upper bound for ingestion retry attempts (review H2): bounds worst-case backoff
52
+ # stall. 10 attempts = 9 exponential backoffs/provider (1.5,3,6,12,24, then capped
53
+ # at 30s → ~167s/provider) × 4 providers ≈ 11 min worst case — below the cron budget.
54
+ _MAX_INGESTION_ATTEMPTS_CAP = 10
55
+
56
+
57
+ def _env_int(name: str, default: int) -> int:
58
+ """Parse an int env var, falling back to `default` on empty/malformed values.
59
+
60
+ Review H1: `int(os.getenv(name, "3"))` crashed on a set-but-empty var
61
+ (`export X=`) or a typo. Fail soft to the default + log, never crash the
62
+ pipeline factory over a bad env value.
63
+ """
64
+ raw = os.getenv(name)
65
+ try:
66
+ return int(raw) if raw not in (None, "") else default
67
+ except (TypeError, ValueError):
68
+ LOGGER.warning("env %s=%r is not a valid int; falling back to %d", name, raw, default)
69
+ return default
70
+
71
+
72
+ def _env_float(name: str, default: float) -> float:
73
+ raw = os.getenv(name)
74
+ try:
75
+ return float(raw) if raw not in (None, "") else default
76
+ except (TypeError, ValueError):
77
+ LOGGER.warning("env %s=%r is not a valid float; falling back to %g", name, raw, default)
78
+ return default
79
+
80
+ SupportedTranscriptSource = TranscriptSource
81
+ DEFAULT_TARGET_AGENTS = ["codex", "claude-code", "openclaw", "opencode"]
82
+ LOGGER = logging.getLogger(__name__)
83
+ DEFAULT_WHISPER_CPP_BINARY = "/usr/local/bin/whisper-cli"
84
+ DEFAULT_WHISPER_CPP_MODEL = str(Path.home() / ".openclaw/models/whisper/ggml-base.bin")
85
+
86
+
87
+ class DeterministicEmbeddingClient:
88
+ def __init__(self, dimensions: int = 12) -> None:
89
+ self._dimensions = dimensions
90
+
91
+ @property
92
+ def model_name(self) -> str:
93
+ return "deterministic-local"
94
+
95
+ def encode(self, texts: Sequence[str]) -> list[list[float]]:
96
+ return [self._encode_text(text) for text in texts]
97
+
98
+ def _encode_text(self, text: str) -> list[float]:
99
+ vector = [0.0] * self._dimensions
100
+ words = re.findall(r"[a-z0-9]+", text.casefold())
101
+ if not words:
102
+ return vector
103
+
104
+ for word in words:
105
+ index = sum(ord(char) for char in word) % self._dimensions
106
+ vector[index] += 1.0
107
+ return vector
108
+
109
+
110
+ class FixtureAwareIngestionFacade:
111
+ def __init__(self, settings: AppSettings | None = None) -> None:
112
+ self._settings = settings or AppSettings()
113
+ self._live_ingestion: IngestionService | None = None
114
+ from ytk.ingestion.transcript_cache import TranscriptCache
115
+
116
+ self.transcript_cache = TranscriptCache(
117
+ cache_dir=self._settings.transcript_cache_dir,
118
+ enabled=self._settings.transcript_cache_enabled,
119
+ )
120
+
121
+ def fetch_from_url(
122
+ self,
123
+ url: str,
124
+ *,
125
+ fixture_transcript: str | Path | None = None,
126
+ ) -> TranscriptBundle:
127
+ parsed_url = parse_youtube_url(url)
128
+ if fixture_transcript is not None:
129
+ # Fixtures are already local — caching them would only shadow the
130
+ # fixture source with a stale copy.
131
+ return self._fixture_bundle(parsed_url, Path(fixture_transcript))
132
+ languages = tuple(self._settings.transcript_languages or ("en",))
133
+ cached = self.transcript_cache.get(parsed_url.video_id, languages)
134
+ if cached is not None:
135
+ return cached
136
+ bundle = self._live_ingestion_service().fetch(parsed_url)
137
+ self.transcript_cache.put(bundle, languages)
138
+ return bundle
139
+
140
+ def _fixture_bundle(self, parsed_url: ParsedYouTubeUrl, fixture_path: Path) -> TranscriptBundle:
141
+ try:
142
+ payload = json.loads(fixture_path.read_text(encoding="utf-8"))
143
+ except FileNotFoundError as exc:
144
+ raise UserFacingError(f"Fixture transcript not found: {fixture_path}") from exc
145
+ except json.JSONDecodeError as exc:
146
+ raise UserFacingError(f"Fixture transcript is not valid JSON: {fixture_path}") from exc
147
+
148
+ transcript_source = self._infer_transcript_source(fixture_path)
149
+ metadata = self._base_metadata(parsed_url)
150
+ metadata.update(
151
+ {
152
+ "title": metadata.get("title") or f"Fixture transcript for {parsed_url.video_id}",
153
+ "provider_name": "fixture",
154
+ }
155
+ )
156
+ segments = [
157
+ TranscriptSegment(
158
+ text=str(item["text"]).strip(),
159
+ raw_text=str(item["text"]).strip(),
160
+ cleaned_text=None,
161
+ start=float(item["start"]),
162
+ end=self._segment_end(item),
163
+ source=transcript_source,
164
+ )
165
+ for item in payload
166
+ ]
167
+ return TranscriptBundle(
168
+ video_id=parsed_url.video_id,
169
+ canonical_url=parsed_url.canonical_url,
170
+ transcript_source=transcript_source,
171
+ segments=segments,
172
+ metadata=metadata,
173
+ )
174
+
175
+ def _live_ingestion_service(self) -> IngestionService:
176
+ if self._live_ingestion is None:
177
+ metadata_fetcher = self._build_production_metadata_fetcher()
178
+ transcript_proxy = TranscriptApiProvider.build_proxy_config(self._settings)
179
+ transcript_languages = self._settings.transcript_languages or ("en",)
180
+ self._live_ingestion = IngestionService(
181
+ caption_providers=[
182
+ TranscriptApiProvider(
183
+ metadata_fetcher=metadata_fetcher,
184
+ languages=transcript_languages,
185
+ proxy_config=transcript_proxy,
186
+ ),
187
+ ],
188
+ subtitle_fallback_provider=YtDlpSubtitleProvider(
189
+ metadata_fetcher=metadata_fetcher,
190
+ language=transcript_languages[0],
191
+ fallback_languages=tuple(transcript_languages[1:]),
192
+ proxy_url=self._settings.transcript_http_proxy_url,
193
+ cookies_file=self._settings.yt_dlp_cookies_file,
194
+ cookies_from_browser=self._settings.yt_dlp_cookies_from_browser,
195
+ ),
196
+ audio_fallback_provider=WhisperCppAsrProvider(
197
+ metadata_fetcher=metadata_fetcher,
198
+ whisper_cpp_binary=self._settings.whisper_cpp_binary or DEFAULT_WHISPER_CPP_BINARY,
199
+ model_path=self._settings.whisper_cpp_model or DEFAULT_WHISPER_CPP_MODEL,
200
+ language=self._settings.whisper_cpp_language,
201
+ ffmpeg_binary=self._settings.ffmpeg_binary,
202
+ yt_dlp_binary=self._settings.yt_dlp_binary,
203
+ cookies_file=self._settings.yt_dlp_cookies_file,
204
+ cookies_from_browser=self._settings.yt_dlp_cookies_from_browser,
205
+ ),
206
+ minimax_fallback_provider=self._build_minimax_provider(metadata_fetcher) if self._settings.minimax_api_key else None,
207
+ # Retry/backoff (§4.1): production opts in here. Env-overridable so a
208
+ # flaky 01:00 cron can be tuned without a code change. Default 3
209
+ # attempts, 1.5s base (exponential ×2 + jitter, capped 30s). Safe
210
+ # parse + cap (review H1/H2): bad/empty env won't crash the factory
211
+ # and attempts can't stall the cron for hours.
212
+ max_attempts=min(_env_int("YTK_INGESTION_MAX_ATTEMPTS", 3), _MAX_INGESTION_ATTEMPTS_CAP),
213
+ base_delay=_env_float("YTK_INGESTION_BASE_DELAY", 1.5),
214
+ )
215
+ return self._live_ingestion
216
+
217
+ def _build_minimax_provider(self, metadata_fetcher: Callable) -> MiniMaxVideoProvider | None:
218
+ from ytk.ingestion.providers.minimax import MiniMaxVideoProvider
219
+
220
+ return MiniMaxVideoProvider(
221
+ metadata_fetcher=metadata_fetcher,
222
+ api_key=self._settings.minimax_api_key,
223
+ region=self._settings.minimax_region,
224
+ model=self._settings.minimax_model,
225
+ max_tokens=self._settings.minimax_max_tokens,
226
+ video_detail=self._settings.minimax_video_detail,
227
+ video_fps=self._settings.minimax_video_fps,
228
+ max_long_side_pixel=self._settings.minimax_max_long_side_pixel,
229
+ max_video_mb=self._settings.minimax_max_video_mb,
230
+ yt_dlp_binary=self._settings.yt_dlp_binary,
231
+ cookies_file=self._settings.yt_dlp_cookies_file,
232
+ cookies_from_browser=self._settings.yt_dlp_cookies_from_browser,
233
+ )
234
+
235
+ @staticmethod
236
+ def _segment_end(item: dict[str, Any]) -> float:
237
+ if "end" in item:
238
+ return float(item["end"])
239
+ duration = float(item.get("duration", 0.0))
240
+ return float(item["start"]) + duration
241
+
242
+ @staticmethod
243
+ def _infer_transcript_source(fixture_path: Path) -> SupportedTranscriptSource:
244
+ name = fixture_path.stem.casefold()
245
+ if "minimax" in name:
246
+ return "minimax"
247
+ if "asr" in name:
248
+ return "asr"
249
+ if "auto" in name:
250
+ return "auto"
251
+ if "yt" in name:
252
+ return "yt_dlp"
253
+ return "manual"
254
+
255
+ def _build_production_metadata_fetcher(self) -> Callable[[ParsedYouTubeUrl], Mapping[str, object]]:
256
+ """Enrich the base metadata stub with yt-dlp (title/duration/chapters).
257
+
258
+ Roadmap 2026-10-06 §3.1.4 (bd zui): without this step the production
259
+ primary provider never saw chapters, so chapter-aligned segmentation
260
+ (``segmentation_source: "chapter"``) and the hour-aware citation
261
+ contract were dead code paths. yt-dlp is an existing dependency; the
262
+ metadata-only network step was approved via roadmap §8.6 + the user's
263
+ 2026-10-06 implementation authorization. Failure degrades to the bare
264
+ stub inside YtDlpMetadataFetcher — transcription still proceeds.
265
+ """
266
+ from ytk.ingestion.providers.ytdlp import YtDlpMetadataFetcher
267
+
268
+ return YtDlpMetadataFetcher(
269
+ fallback=self._base_metadata,
270
+ proxy_url=self._settings.transcript_http_proxy_url,
271
+ cookies_file=self._settings.yt_dlp_cookies_file,
272
+ cookies_from_browser=self._settings.yt_dlp_cookies_from_browser,
273
+ )
274
+
275
+ @staticmethod
276
+ def _base_metadata(parsed_url: ParsedYouTubeUrl) -> dict[str, object]:
277
+ return {
278
+ "video_id": parsed_url.video_id,
279
+ "canonical_url": parsed_url.canonical_url,
280
+ "title": f"YouTube video {parsed_url.video_id}",
281
+ }
282
+
283
+
284
+ class ExtractionPipeline:
285
+ def __init__(
286
+ self,
287
+ ingestion: FixtureAwareIngestionFacade,
288
+ cleaning: CleaningService,
289
+ segmentation: TopicSegmentationService,
290
+ extraction: ExtractionService,
291
+ dedup: DeduplicationService,
292
+ renderers: dict[str, Renderer],
293
+ skill_builder: SkillBuilder,
294
+ target_agents: Sequence[str] | None = None,
295
+ ) -> None:
296
+ self.ingestion = ingestion
297
+ self.cleaning = cleaning
298
+ self.segmentation = segmentation
299
+ self.extraction = extraction
300
+ self.dedup = dedup
301
+ self.renderers = dict(renderers)
302
+ self.skill_builder = skill_builder
303
+ self.target_agents = list(target_agents or DEFAULT_TARGET_AGENTS)
304
+
305
+ def run(
306
+ self,
307
+ *,
308
+ url: str,
309
+ output_dir: Path,
310
+ formats: set[str],
311
+ fixture_transcript: str | Path | None = None,
312
+ doc_language: str | None = None,
313
+ with_reviewer_scaffolding: bool = False,
314
+ ) -> ExtractionBundle:
315
+ manifest = self._manifest_for(output_dir)
316
+ transcript_bundle = self.ingestion.fetch_from_url(url, fixture_transcript=fixture_transcript)
317
+ if fixture_transcript is not None and manifest.metadata_json.exists():
318
+ # A local transcript fixture is not a source of real video metadata.
319
+ # Preserve existing metadata byte-for-byte, but only when its video
320
+ # identity is unambiguous and matches the requested video.
321
+ try:
322
+ existing = json.loads(manifest.metadata_json.read_text(encoding="utf-8"))
323
+ if (not isinstance(existing, dict) or existing.get("video_id") != transcript_bundle.video_id
324
+ or not isinstance(existing.get("title"), str) or not existing["title"].strip()):
325
+ raise ValueError("invalid or mismatched existing video metadata")
326
+ canonical = existing.get("canonical_url")
327
+ if canonical is not None and (not isinstance(canonical, str)
328
+ or parse_youtube_url(canonical).video_id != transcript_bundle.video_id):
329
+ raise ValueError("existing metadata URL does not match video")
330
+ except (OSError, UnicodeError, ValueError, TypeError, UserFacingError) as exc:
331
+ raise RuntimeError("fixture cannot replace invalid or mismatched existing metadata") from exc
332
+ transcript_bundle = transcript_bundle.model_copy(update={"metadata": existing})
333
+ else:
334
+ self._persist_stage_json(manifest.metadata_json, transcript_bundle.metadata)
335
+ self._persist_stage_json(manifest.raw_transcript_json, transcript_bundle)
336
+
337
+ cleaned = self.cleaning.clean(transcript_bundle.segments)
338
+ self._persist_stage_json(manifest.cleaned_transcript_json, cleaned)
339
+
340
+ raw_chapters = transcript_bundle.metadata.get("chapters")
341
+ chapters = raw_chapters if isinstance(raw_chapters, list) else None
342
+ topics = self.segmentation.segment(cleaned, chapters=chapters)
343
+ self._persist_stage_json(manifest.topic_segments_json, topics)
344
+
345
+ atoms = self._extract_atoms(topics, cleaned, transcript_bundle.video_id)
346
+ merged = self.dedup.merge(atoms)
347
+ bundle = self._build_bundle(transcript_bundle, topics, merged)
348
+ self._persist_outputs(
349
+ bundle,
350
+ manifest,
351
+ formats,
352
+ doc_language=doc_language,
353
+ with_reviewer_scaffolding=with_reviewer_scaffolding,
354
+ )
355
+ return bundle
356
+
357
+ def _extract_atoms(
358
+ self,
359
+ topics: list[TopicSegment],
360
+ cleaned_segments: list[TranscriptSegment],
361
+ source_video: str,
362
+ ) -> list[KnowledgeAtom]:
363
+ if not topics:
364
+ return self.extraction.extract(cleaned_segments, ["Transcript"], source_video)
365
+
366
+ atoms: list[KnowledgeAtom] = []
367
+ for topic in topics:
368
+ topic_segments = self._segments_for_topic(cleaned_segments, topic)
369
+ if not topic_segments:
370
+ continue
371
+ atoms.extend(
372
+ self.extraction.extract(
373
+ topic_segments,
374
+ topic_path=[topic.title, topic.id],
375
+ source_video=source_video,
376
+ )
377
+ )
378
+ return atoms
379
+
380
+ @staticmethod
381
+ def _segments_for_topic(
382
+ cleaned_segments: Sequence[TranscriptSegment],
383
+ topic: TopicSegment,
384
+ ) -> list[TranscriptSegment]:
385
+ return [
386
+ segment
387
+ for segment in cleaned_segments
388
+ if segment.start >= topic.start and segment.start < topic.end
389
+ ]
390
+
391
+ def _build_bundle(
392
+ self,
393
+ transcript_bundle: TranscriptBundle,
394
+ topics: list[TopicSegment],
395
+ knowledge_atoms: list[KnowledgeAtom],
396
+ ) -> ExtractionBundle:
397
+ metadata = dict(transcript_bundle.metadata)
398
+ metadata.setdefault("canonical_url", transcript_bundle.canonical_url)
399
+ metadata.setdefault("title", f"YouTube video {transcript_bundle.video_id}")
400
+ return ExtractionBundle(
401
+ video_id=transcript_bundle.video_id,
402
+ metadata=metadata,
403
+ transcript_source=transcript_bundle.transcript_source,
404
+ topic_segments=topics,
405
+ knowledge_atoms=knowledge_atoms,
406
+ skill_spec=self._build_skill_spec(transcript_bundle, topics),
407
+ )
408
+
409
+ def _build_skill_spec(
410
+ self,
411
+ transcript_bundle: TranscriptBundle,
412
+ topics: list[TopicSegment],
413
+ ) -> SkillSpec:
414
+ title = str(transcript_bundle.metadata.get("title") or transcript_bundle.video_id)
415
+ return SkillSpec(
416
+ skill_name=self._slugify(title) or f"youtube-video-{transcript_bundle.video_id}",
417
+ purpose="Turn a tutorial transcript into a reusable, evidence-grounded skill package.",
418
+ trigger_description=(
419
+ "Example output generated deterministically by "
420
+ "knowledge-distiller from this source video — a provenance "
421
+ "demo of the skill format, not an installable standalone skill."
422
+ ),
423
+ target_agents=list(self.target_agents),
424
+ required_files=["SKILL.md", "references/source-summary.md"],
425
+ generated_files=[
426
+ "references/commands.md",
427
+ "references/configs.md",
428
+ "references/concepts.md",
429
+ "examples/example-output.md",
430
+ ],
431
+ references_map={topic.id: topic.title for topic in topics},
432
+ )
433
+
434
+ @staticmethod
435
+ def _slugify(value: str) -> str:
436
+ slug = re.sub(r"[^a-z0-9]+", "-", value.casefold()).strip("-")
437
+ return slug[:80]
438
+
439
+ @staticmethod
440
+ def _manifest_for(output_dir: Path) -> ArtifactManifest:
441
+ output_dir = Path(output_dir)
442
+ output_dir.mkdir(parents=True, exist_ok=True)
443
+ return ArtifactManifest(
444
+ root=output_dir,
445
+ metadata_json=output_dir / "metadata.json",
446
+ raw_transcript_json=output_dir / "raw-transcript.json",
447
+ cleaned_transcript_json=output_dir / "cleaned-transcript.json",
448
+ topic_segments_json=output_dir / "topic-segments.json",
449
+ knowledge_json=output_dir / "knowledge.json",
450
+ notes_md=output_dir / "notes.md",
451
+ knowledge_summary_md=output_dir / "summary.md",
452
+ obsidian_md=output_dir / "obsidian.md",
453
+ skill_dir=output_dir / "skill",
454
+ )
455
+
456
+ @staticmethod
457
+ def _persist_stage_json(output_path: Path, payload: Any) -> None:
458
+ if hasattr(payload, "model_dump"):
459
+ serializable = payload.model_dump(mode="json")
460
+ elif isinstance(payload, list):
461
+ serializable = [
462
+ item.model_dump(mode="json") if hasattr(item, "model_dump") else item for item in payload
463
+ ]
464
+ else:
465
+ serializable = payload
466
+
467
+ output_path.write_text(
468
+ json.dumps(serializable, indent=2, ensure_ascii=False) + "\n",
469
+ encoding="utf-8",
470
+ )
471
+
472
+ def _persist_outputs(
473
+ self,
474
+ bundle: ExtractionBundle,
475
+ manifest: ArtifactManifest,
476
+ formats: set[str],
477
+ doc_language: str | None = None,
478
+ with_reviewer_scaffolding: bool = False,
479
+ ) -> None:
480
+ unsupported = formats - self.renderers.keys()
481
+ if unsupported:
482
+ supported = ", ".join(sorted(self.renderers))
483
+ unknown = ", ".join(sorted(unsupported))
484
+ raise ValueError(f"Unsupported format(s): {unknown}. Expected one of: {supported}")
485
+
486
+ format_to_path: dict[str, Path] = {
487
+ "json": manifest.knowledge_json,
488
+ "jsonl": manifest.root / "knowledge.jsonl",
489
+ "md": manifest.notes_md,
490
+ "obsidian": manifest.obsidian_md,
491
+ "summary": manifest.knowledge_summary_md,
492
+ "doc": manifest.root / "knowledge.md",
493
+ }
494
+ # Build per-format render kwargs (only `doc` currently accepts the
495
+ # reviewer-scaffolding flag; other renderers ignore it).
496
+ render_kwargs: dict[str, dict[str, Any]] = {
497
+ "doc": {"with_reviewer_scaffolding": with_reviewer_scaffolding},
498
+ }
499
+ if "skill" in formats:
500
+ # v1 semantics (A01): the legacy path always builds the provenance
501
+ # package; NOT_APPLICABLE refusal is a v2-surface behavior only.
502
+ self.renderers["skill"].render(
503
+ bundle,
504
+ manifest.skill_dir,
505
+ package_kind="generic",
506
+ require_actionable=False,
507
+ )
508
+ with self._doc_language_scope(doc_language):
509
+ # Preserve independent non-doc artifacts even if the final doc is
510
+ # degraded; never report the overall extraction as complete then.
511
+ for fmt in sorted(formats, key=lambda value: (value == "doc", value)):
512
+ if fmt == "skill":
513
+ continue # handled above with kwargs
514
+ if fmt in format_to_path:
515
+ renderer = self.renderers[fmt]
516
+ renderer.render(
517
+ bundle, format_to_path[fmt], **render_kwargs.get(fmt, {})
518
+ )
519
+ if fmt == "doc" and isinstance(renderer, KnowledgeDocRenderer):
520
+ if not complete_doc_status(manifest.root):
521
+ message = "knowledge document is degraded or unverified; see .knowledge-doc-status.json"
522
+ if os.environ.get("YTK_REQUIRE_COMPLETE_DOC", "").strip().lower() in {"1", "true", "yes", "on"}:
523
+ raise RuntimeError(message)
524
+ LOGGER.warning(message)
525
+
526
+ def _doc_language_scope(self, doc_language: str | None):
527
+ """Context manager: temporarily swap the doc renderer for the requested language.
528
+
529
+ Exposed as a separate method (not just an inline block) so unit tests
530
+ can exercise the swap without standing up the full pipeline.
531
+ """
532
+ from contextlib import contextmanager
533
+
534
+ from ytk.renderers.knowledge_doc_renderer import KnowledgeDocRenderer
535
+
536
+ @contextmanager
537
+ def _scope():
538
+ if not (doc_language and "doc" in self.renderers):
539
+ yield
540
+ return
541
+ original = self.renderers["doc"]
542
+ self.renderers["doc"] = KnowledgeDocRenderer(language=doc_language)
543
+ try:
544
+ yield
545
+ finally:
546
+ self.renderers["doc"] = original
547
+
548
+ return _scope()
549
+
550
+
551
+ def build_default_pipeline(settings: AppSettings | None = None) -> ExtractionPipeline:
552
+ settings = settings or AppSettings()
553
+ skill_builder = SkillBuilder()
554
+ return ExtractionPipeline(
555
+ ingestion=FixtureAwareIngestionFacade(settings),
556
+ cleaning=CleaningService(),
557
+ segmentation=TopicSegmentationService(
558
+ embedding_client=_build_default_embedding_client(settings),
559
+ ),
560
+ extraction=ExtractionService(default_strategy=HeuristicExtractor()),
561
+ dedup=DeduplicationService(),
562
+ renderers={
563
+ "json": JsonRenderer(),
564
+ "jsonl": JSONLRenderer(),
565
+ "md": MarkdownRenderer(),
566
+ "obsidian": ObsidianRenderer(),
567
+ "summary": SummaryRenderer(),
568
+ "doc": KnowledgeDocRenderer(),
569
+ "skill": SkillPackageRenderer(builder=skill_builder),
570
+ },
571
+ skill_builder=skill_builder,
572
+ )
573
+
574
+
575
+ def _build_default_embedding_client(settings: AppSettings) -> SentenceTransformerClient | DeterministicEmbeddingClient:
576
+ try:
577
+ importlib.import_module("sentence_transformers")
578
+ return SentenceTransformerClient(model_name=settings.sentence_transformer_model)
579
+ except Exception as exc:
580
+ LOGGER.warning(
581
+ "Falling back to deterministic embeddings because sentence-transformers is unavailable: %s",
582
+ exc,
583
+ )
584
+ return DeterministicEmbeddingClient()
585
+
586
+
587
+ def load_bundle(path: Path) -> ExtractionBundle:
588
+ try:
589
+ payload = json.loads(path.read_text(encoding="utf-8"))
590
+ except FileNotFoundError as exc:
591
+ raise UserFacingError(f"Knowledge bundle not found: {path}") from exc
592
+ except json.JSONDecodeError as exc:
593
+ raise UserFacingError(f"Knowledge bundle is not valid JSON: {path}") from exc
594
+
595
+ try:
596
+ return ExtractionBundle.model_validate(payload)
597
+ except ValidationError as exc:
598
+ raise UserFacingError(f"Failed to load knowledge bundle: {exc}") from exc
@@ -0,0 +1,18 @@
1
+ """Implementation identity for extraction judgments and their rendered output."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ from pathlib import Path
8
+
9
+
10
+ def extraction_implementation_fingerprint(package_root: Path | None = None) -> str:
11
+ root = package_root or Path(__file__).resolve().parents[1]
12
+ paths = [*sorted((root / "extraction").rglob("*.py")), root / "text_normalization.py"]
13
+ digests = {}
14
+ for path in paths:
15
+ with path.open("rb") as handle:
16
+ digests[path.relative_to(root).as_posix()] = hashlib.file_digest(handle, "sha256").hexdigest()
17
+ payload = json.dumps(digests, sort_keys=True, separators=(",", ":")).encode()
18
+ return "sha256:" + hashlib.sha256(payload).hexdigest()