kd-engine 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. kd_engine-1.0.0.dist-info/METADATA +19 -0
  2. kd_engine-1.0.0.dist-info/RECORD +118 -0
  3. kd_engine-1.0.0.dist-info/WHEEL +5 -0
  4. kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
  5. kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
  6. kd_engine-1.0.0.dist-info/top_level.txt +1 -0
  7. ytk/__init__.py +2 -0
  8. ytk/__main__.py +21 -0
  9. ytk/batch/__init__.py +39 -0
  10. ytk/batch/runner.py +1149 -0
  11. ytk/cache/__init__.py +34 -0
  12. ytk/cache/v2.py +562 -0
  13. ytk/cleaning/density.py +40 -0
  14. ytk/cleaning/fillers.py +18 -0
  15. ytk/cleaning/sentence_boundary.py +30 -0
  16. ytk/cleaning/service.py +62 -0
  17. ytk/cli.py +1281 -0
  18. ytk/commands/__init__.py +3 -0
  19. ytk/commands/merge.py +600 -0
  20. ytk/config.py +757 -0
  21. ytk/dedup/service.py +166 -0
  22. ytk/doctor.py +891 -0
  23. ytk/dotenv.py +334 -0
  24. ytk/errors.py +38 -0
  25. ytk/extraction/service.py +451 -0
  26. ytk/extraction/strategies/base.py +16 -0
  27. ytk/extraction/strategies/code_block.py +285 -0
  28. ytk/extraction/strategies/heuristic.py +791 -0
  29. ytk/extraction/strategies/local_model.py +56 -0
  30. ytk/extraction/text_context.py +177 -0
  31. ytk/extraction/visual_context.py +159 -0
  32. ytk/ingestion/adapters/__init__.py +1 -0
  33. ytk/ingestion/adapters/_common.py +865 -0
  34. ytk/ingestion/adapters/base.py +160 -0
  35. ytk/ingestion/adapters/bilibili.py +271 -0
  36. ytk/ingestion/adapters/douyin.py +445 -0
  37. ytk/ingestion/adapters/tiktok.py +102 -0
  38. ytk/ingestion/adapters/xiaohongshu.py +758 -0
  39. ytk/ingestion/adapters/youtube.py +74 -0
  40. ytk/ingestion/browser_acquired.py +220 -0
  41. ytk/ingestion/http_client.py +43 -0
  42. ytk/ingestion/local_import.py +202 -0
  43. ytk/ingestion/media.py +468 -0
  44. ytk/ingestion/providers/asr.py +902 -0
  45. ytk/ingestion/providers/base.py +13 -0
  46. ytk/ingestion/providers/dtk_client.py +816 -0
  47. ytk/ingestion/providers/minimax.py +350 -0
  48. ytk/ingestion/providers/transcript_api.py +336 -0
  49. ytk/ingestion/providers/xhs_client.py +646 -0
  50. ytk/ingestion/providers/ytdlp.py +400 -0
  51. ytk/ingestion/registry.py +189 -0
  52. ytk/ingestion/resolver.py +417 -0
  53. ytk/ingestion/search.py +56 -0
  54. ytk/ingestion/service.py +167 -0
  55. ytk/ingestion/subtitles.py +602 -0
  56. ytk/ingestion/transcript_cache.py +212 -0
  57. ytk/ingestion/url_parser.py +53 -0
  58. ytk/ingestion/video_frames.py +292 -0
  59. ytk/ingestion/ytdlp_acquire.py +754 -0
  60. ytk/logging.py +383 -0
  61. ytk/models/common.py +20 -0
  62. ytk/models/content.py +451 -0
  63. ytk/models/knowledge.py +43 -0
  64. ytk/models/migrations.py +262 -0
  65. ytk/models/skill.py +13 -0
  66. ytk/models/source.py +265 -0
  67. ytk/models/topic.py +18 -0
  68. ytk/models/transcript.py +30 -0
  69. ytk/ocr/__init__.py +65 -0
  70. ytk/ocr/engine.py +245 -0
  71. ytk/ocr/layout.py +347 -0
  72. ytk/ocr/pipeline.py +440 -0
  73. ytk/ocr/rapidocr_engine.py +144 -0
  74. ytk/pipeline/acquisition_checkpoint.py +240 -0
  75. ytk/pipeline/artifacts.py +18 -0
  76. ytk/pipeline/orchestrator.py +598 -0
  77. ytk/pipeline/processing_identity.py +18 -0
  78. ytk/pipeline/retry.py +728 -0
  79. ytk/pipeline/runners.py +578 -0
  80. ytk/pipeline/state.py +1107 -0
  81. ytk/pipeline/v2.py +1863 -0
  82. ytk/py.typed +1 -0
  83. ytk/quality/__init__.py +9 -0
  84. ytk/quality/completeness.py +260 -0
  85. ytk/renderers/base.py +9 -0
  86. ytk/renderers/citation_alignment.py +617 -0
  87. ytk/renderers/evidence_formatter.py +362 -0
  88. ytk/renderers/json_renderer.py +13 -0
  89. ytk/renderers/jsonl_renderer.py +18 -0
  90. ytk/renderers/knowledge_doc_renderer.py +1984 -0
  91. ytk/renderers/markdown_renderer.py +540 -0
  92. ytk/renderers/obsidian_renderer.py +138 -0
  93. ytk/renderers/skill_package.py +21 -0
  94. ytk/renderers/source_links.py +309 -0
  95. ytk/renderers/summary_renderer.py +163 -0
  96. ytk/renderers/v2.py +171 -0
  97. ytk/renderers/v2_jsonl.py +165 -0
  98. ytk/renderers/v2_markdown.py +477 -0
  99. ytk/renderers/v2_obsidian.py +113 -0
  100. ytk/renderers/v2_summary.py +220 -0
  101. ytk/segmentation/change_points.py +50 -0
  102. ytk/segmentation/embeddings.py +45 -0
  103. ytk/segmentation/service.py +318 -0
  104. ytk/skill_builder/__init__.py +1 -0
  105. ytk/skill_builder/builder.py +419 -0
  106. ytk/skill_builder/templates/SKILL.md.j2 +85 -0
  107. ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
  108. ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
  109. ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
  110. ytk/skill_builder/templates/references/commands.md.j2 +34 -0
  111. ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
  112. ytk/skill_builder/templates/references/configs.md.j2 +31 -0
  113. ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
  114. ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
  115. ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
  116. ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
  117. ytk/skill_builder/templates.py +29 -0
  118. ytk/text_normalization.py +44 -0
@@ -0,0 +1,540 @@
1
+ from __future__ import annotations
2
+
3
+ import re
4
+ from collections import defaultdict
5
+ from pathlib import Path
6
+ from typing import Any
7
+
8
+
9
+ def source_processing_notice(bundle: Any) -> str:
10
+ """Keep acquisition provenance distinct from human review and visual coverage."""
11
+ source = getattr(bundle, "transcript_source", "unknown")
12
+ notice = {
13
+ "manual": "Extracted from manual captions",
14
+ "auto": "Extracted from auto-generated captions; not human reviewed",
15
+ "asr": "Extracted from ASR speech-to-text; not human reviewed",
16
+ "yt_dlp": "Transcript acquired via yt-dlp; caption type not independently verified",
17
+ "minimax": "Cloud-model video interpretation; not human reviewed",
18
+ }.get(source, "Transcript source unknown")
19
+ scope = ("Visual coverage not independently verified (画面覆盖范围未独立验证)"
20
+ if source == "minimax" else "Visuals not analysed (未分析画面); slides and demonstrations may be missing")
21
+ return notice + ". " + scope + "."
22
+
23
+
24
+ class MarkdownRenderer:
25
+ """Render an ExtractionBundle into a clean, human-readable Markdown note."""
26
+
27
+ # ── Garbage-detection patterns ──────────────────────────────────
28
+ _TRUNCATION_ENDINGS = re.compile(
29
+ r"(?:In|The|Once|Now we are|or|and|the|a|in|But|So|That|This|It|We|They"
30
+ r"|You|He|She|What|How|Why|When|Where|Who|Is|Are|Was|Were|Do|Does|Did"
31
+ r"|Can|Could|Will|Would|Should|May|Might|Must|Shall|Has|Have|Had"
32
+ r"|Not|No|Yes|But|Just|Only|Also|Even|Still|Already|Yet|Now|Then"
33
+ r"|Here|There|Well|OK|Okay|Right|Sure|Good|Great|Fine|Nice"
34
+ r"|Like|Just|Really|Very|Much|More|Most|Less|Least|Too|Also)\.$"
35
+ )
36
+ _NATURAL_LANGUAGE_FRAGMENT = re.compile(
37
+ r"^[A-Z][a-z].*\b(?:can|will|is|are|was|were|it|we|you|they|the|this|that)\b"
38
+ )
39
+ _YAML_SPECIAL_RE = re.compile(r"[:#&*!|>%@`]")
40
+
41
+ # ── Public API ─────────────────────────────────────────────────
42
+
43
+ def render(self, bundle: Any, output_path: Path, **kwargs: Any) -> None:
44
+ output_path.parent.mkdir(parents=True, exist_ok=True)
45
+
46
+ video_url = bundle.metadata.get(
47
+ "canonical_url",
48
+ f"https://www.youtube.com/watch?v={bundle.video_id}",
49
+ )
50
+ title = str(bundle.metadata.get("title", bundle.video_id))
51
+
52
+ src = bundle.transcript_source
53
+ clean_atoms = [a for a in bundle.knowledge_atoms if not self._is_garbage(a, source=src)]
54
+
55
+ lines: list[str] = []
56
+
57
+ # ── Header ─────────────────────────────────────────────
58
+ lines.append(f"# {title}")
59
+ lines.append("")
60
+ lines.append(f"> 📺 [Watch on YouTube]({video_url})")
61
+ caption_note = source_processing_notice(bundle)
62
+ lines.append(f"> 🕐 {caption_note}")
63
+ lines.append("")
64
+ lines.append("---")
65
+ lines.append("")
66
+
67
+ # ── Summary (legacy section — tests assert exact format) ──
68
+ lines.append("## Summary")
69
+ lines.append("")
70
+ lines.append(f"- Video ID: `{bundle.video_id}`")
71
+ lines.append(f"- Transcript source: `{bundle.transcript_source}`")
72
+
73
+ canonical_url = bundle.metadata.get("canonical_url")
74
+ if canonical_url:
75
+ lines.append(f"- Canonical URL: {canonical_url}")
76
+
77
+ lines.append("")
78
+
79
+ # ── Overview (new beautiful section) ───────────────────
80
+ overview = self._build_overview(bundle, clean_atoms)
81
+ if overview:
82
+ lines.append("## Overview")
83
+ lines.append("")
84
+ for para in overview:
85
+ lines.append(para)
86
+ lines.append("")
87
+ lines.append("---")
88
+ lines.append("")
89
+
90
+ # ── Executive Summary (legacy — tests assert "stands out in") ──
91
+ executive_summary = self._build_executive_summary(bundle, clean_atoms)
92
+ if executive_summary:
93
+ lines.append("## Executive Summary")
94
+ lines.append("")
95
+ for line in executive_summary:
96
+ lines.append(f"- {line}")
97
+ lines.append("")
98
+
99
+ # ── Key Takeaways (legacy — tests assert this structure) ──
100
+ grouped = self._group_atoms(clean_atoms)
101
+ if any(grouped.values()):
102
+ lines.append("## Key Takeaways")
103
+ lines.append("")
104
+ self._append_group(
105
+ lines,
106
+ "Commands",
107
+ grouped["terminal_command"],
108
+ formatter=lambda atom: f"- `{atom.content}`",
109
+ )
110
+ self._append_group(
111
+ lines,
112
+ "Files & Config",
113
+ grouped["configuration_snippet"],
114
+ formatter=lambda atom: f"- `{atom.content}`",
115
+ )
116
+ self._append_group(
117
+ lines,
118
+ "Concepts",
119
+ grouped["core_concept"],
120
+ formatter=lambda atom: f"- `{atom.content}`: {self._display_explanation(atom)}",
121
+ )
122
+ self._append_group(
123
+ lines,
124
+ "Workflows",
125
+ grouped["workflow_step"] + grouped["best_practice"],
126
+ formatter=lambda atom: f"- {atom.content}",
127
+ )
128
+ self._append_group(
129
+ lines,
130
+ "Code Blocks",
131
+ grouped["code_block"],
132
+ formatter=lambda atom: f"- `{atom.content}`",
133
+ )
134
+ self._append_group(
135
+ lines,
136
+ "Command Sequences",
137
+ grouped["command_sequence"],
138
+ formatter=lambda atom: f"- `{atom.content}`",
139
+ )
140
+
141
+ # ── Topic Notes (legacy — tests assert atom details) ───
142
+ lines.append("## Topic Notes")
143
+ lines.append("")
144
+
145
+ atoms_by_topic = defaultdict(list)
146
+ for atom in clean_atoms:
147
+ topic_key = self._topic_key_from_path(atom.topic_path)
148
+ atoms_by_topic[topic_key].append(atom)
149
+
150
+ rendered_topics: set[str] = set()
151
+ for topic in bundle.topic_segments:
152
+ rendered_topics.add(topic.id)
153
+ lines.append(f"## {topic.title} ({topic.start:.1f}s–{topic.end:.1f}s)")
154
+ lines.append("")
155
+ if topic.sentences:
156
+ lines.append(f"- Summary: {topic.sentences[0]}")
157
+ lines.append("")
158
+
159
+ for atom in atoms_by_topic.get(topic.id, []):
160
+ lines.append(
161
+ f"### {atom.atom_type.replace('_', ' ').title()}: {atom.content}"
162
+ )
163
+ lines.append(f"- Confidence: {atom.confidence:.2f}")
164
+ lines.append(f"- Explanation: {atom.explanation}")
165
+ for evidence in atom.evidence:
166
+ lines.append(
167
+ f"- Evidence: `{evidence.text}` "
168
+ f"({evidence.start:.1f}s-{evidence.end:.1f}s, {evidence.source})"
169
+ )
170
+ for timestamp in atom.timestamps:
171
+ lines.append(
172
+ f"- Watch: https://www.youtube.com/watch"
173
+ f"?v={bundle.video_id}&t={int(timestamp)}s"
174
+ )
175
+ lines.append("")
176
+
177
+ for topic_key, atoms in atoms_by_topic.items():
178
+ if topic_key in rendered_topics:
179
+ continue
180
+ display_name = (
181
+ atoms[0].topic_path[0] if atoms and atoms[0].topic_path else "Ungrouped"
182
+ )
183
+ lines.append(f"## {display_name}")
184
+ lines.append("")
185
+ for atom in atoms:
186
+ lines.append(
187
+ f"### {atom.atom_type.replace('_', ' ').title()}: {atom.content}"
188
+ )
189
+ lines.append(f"- Confidence: {atom.confidence:.2f}")
190
+ lines.append(f"- Explanation: {atom.explanation}")
191
+ for evidence in atom.evidence:
192
+ lines.append(
193
+ f"- Evidence: `{evidence.text}` "
194
+ f"({evidence.start:.1f}s-{evidence.end:.1f}s, {evidence.source})"
195
+ )
196
+ for timestamp in atom.timestamps:
197
+ lines.append(
198
+ f"- Watch: https://www.youtube.com/watch"
199
+ f"?v={bundle.video_id}&t={int(timestamp)}s"
200
+ )
201
+ lines.append("")
202
+
203
+ # ── Stats footer ───────────────────────────────────────
204
+ lines.append("")
205
+ lines.append("---")
206
+ lines.append("")
207
+ lines.append("## Stats")
208
+ lines.append("")
209
+ lines.append(f"- Total knowledge atoms: {len(bundle.knowledge_atoms)}")
210
+ type_counts: dict[str, int] = {}
211
+ for atom in bundle.knowledge_atoms:
212
+ type_counts[atom.atom_type] = type_counts.get(atom.atom_type, 0) + 1
213
+ for atype, count in sorted(type_counts.items()):
214
+ lines.append(f"- {atype.replace('_', ' ').title()}: {count}")
215
+
216
+ lines.append("")
217
+ lines.append(
218
+ f"*Generated by knowledge-distiller v2.0"
219
+ f" • {len(bundle.knowledge_atoms)} knowledge atoms extracted*"
220
+ )
221
+
222
+ output_path.write_text("\n".join(lines), encoding="utf-8")
223
+
224
+ # ── Sanitization helpers ───────────────────────────────────────
225
+
226
+ @classmethod
227
+ def _is_garbage(cls, atom: Any, *, source: str = "auto") -> bool:
228
+ """Return True when an atom is noise that shouldn't appear in output.
229
+
230
+ Universal checks (all sources):
231
+ - Very low confidence
232
+ - && join artifacts from heuristic extractor
233
+ - Terminal commands with slash prefix + boilerplate explanation
234
+ - Workflow steps with heuristic template explanation
235
+ - Core concepts with truncated descriptions
236
+
237
+ Auto/ASR only checks:
238
+ - Mid-sentence fragments (lowercase start)
239
+ - Too few words
240
+ - Truncated caption endings
241
+ - Natural language command_sequence fragments
242
+
243
+ Note on "template" rules: an atom with a heuristic-style template
244
+ explanation (e.g. "This sentence expresses an ordered workflow
245
+ instruction.") is only treated as noise when it has no transcript
246
+ evidence attached. Real tutorial extractions always carry evidence
247
+ with a start/end timestamp pointing back to the source caption, so
248
+ keeping those atoms preserves the most useful tutorial knowledge
249
+ rather than silently dropping it. v2.0 originally filtered these
250
+ unconditionally; that turned out to discard the bulk of the
251
+ workflow_step and command_sequence atoms from real YouTube
252
+ tutorial transcripts (validated on vRYBG_R8JAI, 2026-06-07).
253
+ """
254
+ content = str(atom.content).strip()
255
+ explanation = str(getattr(atom, "explanation", "")).strip()
256
+ has_evidence = bool(getattr(atom, "evidence", None))
257
+
258
+ # ── Universal garbage patterns ──────────────────────────
259
+
260
+ # Confidence too low
261
+ if atom.confidence < 0.5:
262
+ return True
263
+
264
+ # Join artifacts from heuristic extractor (always noise)
265
+ # KEEP if the atom has transcript evidence — heuristic joins are
266
+ # noise when fabricating, but legitimate when each step is grounded
267
+ # in the source transcript.
268
+ if "&&" in content and not has_evidence:
269
+ return True
270
+
271
+ # Heuristic template workflow steps (not real instructions)
272
+ # KEEP if the atom has transcript evidence — same reasoning.
273
+ if (
274
+ atom.atom_type == "workflow_step"
275
+ and explanation == "This sentence expresses an ordered workflow instruction."
276
+ and not has_evidence
277
+ ):
278
+ return True
279
+
280
+ # Core concept with truncated (very short) description
281
+ grounded_chinese = has_evidence and len(re.findall(r"[\u4e00-\u9fff]", content + explanation)) >= 4
282
+ if atom.atom_type == "core_concept" and len(explanation) < 10 and not grounded_chinese:
283
+ return True
284
+
285
+ # Heuristic template core concept (a/an X is a Y) — keep if
286
+ # evidence is present so we surface real tutorial concepts.
287
+ if (
288
+ atom.atom_type == "core_concept"
289
+ and "mentioned in the video that relates to" in explanation
290
+ and not has_evidence
291
+ ):
292
+ return True
293
+
294
+ # ── Auto/ASR only weak-signal patterns ──────────────────
295
+ if source not in ("auto", "asr"):
296
+ return False
297
+
298
+ # Chinese clauses do not use spaces as word boundaries. English word
299
+ # counts otherwise discard every grounded Chinese ASR instruction.
300
+ if grounded_chinese:
301
+ return False
302
+
303
+ # Terminal commands that are auto-caption truncations
304
+ if (
305
+ atom.atom_type == "terminal_command"
306
+ and content.startswith("/")
307
+ and "as shown in the transcript" in explanation
308
+ ):
309
+ return True
310
+
311
+ # Core concept with template-generated explanation (auto captions only)
312
+ if atom.atom_type == "core_concept" and (
313
+ explanation
314
+ == "Important Claude Code workflow concept mentioned in the transcript."
315
+ ):
316
+ return True
317
+ if content and content[0].islower() and atom.atom_type != "terminal_command":
318
+ return True
319
+
320
+ # Too short and not a terminal command
321
+ word_count = len(content.split())
322
+ if word_count < 5 and atom.atom_type != "terminal_command":
323
+ return True
324
+
325
+ # Truncated auto-caption endings
326
+ if cls._TRUNCATION_ENDINGS.search(content):
327
+ return True
328
+
329
+ # Command sequence atoms that are natural language, not commands
330
+ if atom.atom_type == "command_sequence" and cls._NATURAL_LANGUAGE_FRAGMENT.match(
331
+ content
332
+ ):
333
+ return True
334
+
335
+ return False
336
+
337
+ @classmethod
338
+ def clean_content(cls, text: str) -> str:
339
+ """Sanitize a single piece of content text."""
340
+ text = text.strip()
341
+ text = re.sub(r"\s*(?:stands out|is used|is covered|is discussed)\s+in\s+.*$", "", text)
342
+ if text and text[0].islower():
343
+ text = text[0].upper() + text[1:]
344
+ return text
345
+
346
+ @classmethod
347
+ def clean_tag(cls, tag: str) -> str:
348
+ """Produce a clean Obsidian-safe tag."""
349
+ tag = tag.strip()
350
+ tag = tag.replace("\n", " ").replace("\r", " ")
351
+ tag = re.sub(r"[.,;!?()\s]+$", "", tag)
352
+ tag = re.sub(r"^[.,;!?()\s]+", "", tag)
353
+ tag = cls._YAML_SPECIAL_RE.sub("", tag)
354
+ tag = tag.replace(",", "-")
355
+ tag = tag.replace(" ", "-")
356
+ tag = re.sub(r"-+", "-", tag)
357
+ tag = tag.strip("-")
358
+ if re.match(r"^topic-\d+$", tag):
359
+ return ""
360
+ if len(tag) < 3:
361
+ return ""
362
+ if len(tag.split("-")) >= 5:
363
+ return ""
364
+ return tag
365
+
366
+ # ── Overview builder ───────────────────────────────────────────
367
+
368
+ def _build_overview(self, bundle: Any, clean_atoms: list) -> list[str]:
369
+ """Build 2-3 sentence overview from topic segments."""
370
+ paragraphs: list[str] = []
371
+ sentences: list[str] = []
372
+ for topic in bundle.topic_segments:
373
+ for sent in topic.sentences:
374
+ cleaned = self.clean_content(sent)
375
+ if cleaned and len(cleaned.split()) >= 5:
376
+ sentences.append(cleaned)
377
+ if len(sentences) >= 3:
378
+ break
379
+ if len(sentences) >= 3:
380
+ break
381
+ if sentences:
382
+ paragraphs.append(" ".join(sentences))
383
+ return paragraphs
384
+
385
+ if clean_atoms:
386
+ ranked = sorted(clean_atoms, key=lambda a: a.confidence, reverse=True)
387
+ overview_parts = [
388
+ self.clean_content(a.content) for a in ranked[:3] if self.clean_content(a.content)
389
+ ]
390
+ if overview_parts:
391
+ paragraphs.append(" ".join(overview_parts))
392
+ return paragraphs
393
+
394
+ return ["No reliable overview could be extracted from this video's captions."]
395
+
396
+ # ── Executive Summary (keeps "stands out in" for tests) ────────
397
+
398
+ def _build_executive_summary(self, bundle: Any, clean_atoms: list) -> list[str]:
399
+ atoms_by_topic: dict[str, list] = defaultdict(list)
400
+ for atom in clean_atoms:
401
+ topic_key = self._topic_key_from_path(atom.topic_path)
402
+ atoms_by_topic[topic_key].append(atom)
403
+
404
+ summary_lines: list[str] = []
405
+ topic_order = [(t.id, t.title) for t in bundle.topic_segments]
406
+ for topic_key, topic_name in topic_order:
407
+ atoms = atoms_by_topic.get(topic_key, [])
408
+ if not atoms:
409
+ continue
410
+ ranked = sorted(
411
+ atoms,
412
+ key=lambda a: (
413
+ self._atom_priority(a.atom_type),
414
+ self._atom_specificity(a),
415
+ self._atom_summary_rank(a),
416
+ a.confidence,
417
+ ),
418
+ reverse=True,
419
+ )
420
+ if self._topic_is_generic_only(ranked):
421
+ continue
422
+ used: set[str] = set()
423
+ max_items = 2 if self._topic_supports_dense_summary(ranked) else 1
424
+ added = 0
425
+ for atom in ranked:
426
+ key = str(atom.content).casefold()
427
+ if key in used:
428
+ continue
429
+ if self._should_skip_summary_atom(atom, ranked):
430
+ continue
431
+ used.add(key)
432
+ summary_lines.append(self._summary_sentence(topic_name, atom))
433
+ added += 1
434
+ if len(summary_lines) >= 5 or added >= max_items:
435
+ break
436
+ if len(summary_lines) >= 5:
437
+ break
438
+ return summary_lines
439
+
440
+ # ── Legacy helpers (kept for test compatibility) ───────────────
441
+
442
+ @staticmethod
443
+ def _topic_key_from_path(topic_path: list) -> str:
444
+ if len(topic_path) >= 2:
445
+ return str(topic_path[1])
446
+ if topic_path:
447
+ return str(topic_path[0])
448
+ return "Ungrouped"
449
+
450
+ @staticmethod
451
+ def _atom_priority(atom_type: str) -> int:
452
+ return {
453
+ "terminal_command": 5,
454
+ "configuration_snippet": 4,
455
+ "core_concept": 3,
456
+ "best_practice": 2,
457
+ "workflow_step": 1,
458
+ "code_block": 4,
459
+ "command_sequence": 5,
460
+ }.get(atom_type, 0)
461
+
462
+ @staticmethod
463
+ def _atom_specificity(atom: Any) -> int:
464
+ return len(str(atom.content).split())
465
+
466
+ @staticmethod
467
+ def _atom_summary_rank(atom: Any) -> int:
468
+ generic = {"claude code", "terminal", "pattern", "patterns"}
469
+ if atom.atom_type == "core_concept" and str(atom.content).casefold() in generic:
470
+ return 0
471
+ return 1
472
+
473
+ def _topic_supports_dense_summary(self, ranked: list) -> bool:
474
+ specific = [
475
+ a for a in ranked if a.atom_type == "core_concept" and self._atom_summary_rank(a) > 0
476
+ ]
477
+ return len(specific) >= 2
478
+
479
+ def _topic_is_generic_only(self, ranked: list) -> bool:
480
+ if not ranked:
481
+ return True
482
+ return all(
483
+ a.atom_type == "core_concept" and self._atom_summary_rank(a) == 0 for a in ranked
484
+ )
485
+
486
+ def _should_skip_summary_atom(self, atom: Any, ranked: list) -> bool:
487
+ if atom.atom_type != "core_concept":
488
+ return False
489
+ if self._atom_summary_rank(atom) > 0:
490
+ return False
491
+ return any(
492
+ o.atom_type == "core_concept" and self._atom_summary_rank(o) > 0 for o in ranked
493
+ )
494
+
495
+ def _summary_sentence(self, topic_name: str, atom: Any) -> str:
496
+ if atom.atom_type in {"terminal_command", "configuration_snippet"}:
497
+ return f"`{atom.content}` is used in {topic_name}."
498
+ if atom.atom_type == "core_concept":
499
+ return f"{atom.content} is covered in {topic_name}."
500
+ return f"{atom.content} is discussed in {topic_name}."
501
+
502
+ @staticmethod
503
+ def _display_explanation(atom: Any) -> str:
504
+ if (
505
+ atom.atom_type == "core_concept"
506
+ and atom.explanation
507
+ == "Important Claude Code workflow concept mentioned in the transcript."
508
+ ):
509
+ return "highlighted as an important Claude Code concept."
510
+ return atom.explanation
511
+
512
+ @staticmethod
513
+ def _group_atoms(atoms: list) -> dict[str, list]:
514
+ types = [
515
+ "terminal_command",
516
+ "configuration_snippet",
517
+ "core_concept",
518
+ "workflow_step",
519
+ "best_practice",
520
+ "code_block",
521
+ "command_sequence",
522
+ ]
523
+ return {t: [a for a in atoms if a.atom_type == t] for t in types}
524
+
525
+ @staticmethod
526
+ def _append_group(
527
+ lines: list[str], title: str, atoms: list, *, formatter: Any
528
+ ) -> None:
529
+ if not atoms:
530
+ return
531
+ lines.append(f"## {title}")
532
+ lines.append("")
533
+ seen: set[str] = set()
534
+ for atom in atoms:
535
+ rendered = formatter(atom)
536
+ if rendered in seen:
537
+ continue
538
+ seen.add(rendered)
539
+ lines.append(rendered)
540
+ lines.append("")
@@ -0,0 +1,138 @@
1
+ from __future__ import annotations
2
+
3
+ import re
4
+ from pathlib import Path
5
+ from typing import Any
6
+
7
+ from ytk.renderers.markdown_renderer import MarkdownRenderer
8
+
9
+ _YAML_SPECIAL_RE = re.compile(r"[:#&*!|>%@`]")
10
+
11
+
12
+ def _escape_yaml_string(value: str) -> str:
13
+ escaped = value.replace("\\", "\\\\").replace('"', '\\"')
14
+ escaped = escaped.replace("\n", " ")
15
+ return escaped
16
+
17
+
18
+ class ObsidianRenderer:
19
+ """Render an ExtractionBundle as an Obsidian-flavoured Markdown note."""
20
+
21
+ SECTION_TITLES: dict[str, str] = {
22
+ "terminal_command": "## ⌨️ Commands",
23
+ "best_practice": "## ✅ Best Practices",
24
+ "configuration_snippet": "## ⚙️ Configuration",
25
+ "workflow_step": "## 📋 Workflow Steps",
26
+ "core_concept": "## 💡 Core Concepts",
27
+ "code_block": "## 📦 Code Blocks",
28
+ "command_sequence": "## 🔗 Command Sequences",
29
+ }
30
+
31
+ def render(self, bundle: Any, output_path: Path, **kwargs: Any) -> None:
32
+ output_path.parent.mkdir(parents=True, exist_ok=True)
33
+
34
+ video_url = bundle.metadata.get(
35
+ "canonical_url",
36
+ f"https://www.youtube.com/watch?v={bundle.video_id}",
37
+ )
38
+ raw_title = bundle.metadata.get("title", bundle.video_id)
39
+ safe_title = _escape_yaml_string(str(raw_title))
40
+
41
+ clean_atoms = [
42
+ a
43
+ for a in bundle.knowledge_atoms
44
+ if not MarkdownRenderer._is_garbage(a, source=bundle.transcript_source)
45
+ ]
46
+
47
+ lines: list[str] = []
48
+
49
+ # ── YAML frontmatter ───────────────────────────────────
50
+ lines.append("---")
51
+ lines.append(f'title: "{safe_title}"')
52
+ lines.append(f"video_id: {bundle.video_id}")
53
+ if bundle.metadata.get("canonical_url"):
54
+ lines.append(f"url: {bundle.metadata['canonical_url']}")
55
+
56
+ all_tags: set[str] = set()
57
+ for atom in clean_atoms:
58
+ for topic in atom.topic_path:
59
+ tag = MarkdownRenderer.clean_tag(topic)
60
+ if tag:
61
+ all_tags.add(tag)
62
+ if all_tags:
63
+ lines.append("tags:")
64
+ for tag in sorted(all_tags):
65
+ lines.append(f" - {tag}")
66
+ lines.append("---")
67
+ lines.append("")
68
+
69
+ # ── Header ─────────────────────────────────────────────
70
+ lines.append(f"# {raw_title}")
71
+ lines.append("")
72
+ lines.append(f"> 📺 [Watch on YouTube]({video_url})")
73
+ caption_note = (
74
+ "Extracted from auto-generated captions"
75
+ if bundle.transcript_source in ("auto", "asr")
76
+ else "Extracted from manual captions"
77
+ )
78
+ lines.append(f"> 🕐 {caption_note}")
79
+ lines.append("")
80
+ lines.append("---")
81
+ lines.append("")
82
+
83
+ if not clean_atoms:
84
+ lines.append("> ⚠️ No reliable knowledge atoms could be extracted from this video's captions.")
85
+ lines.append("")
86
+
87
+ # ── Grouped sections by atom type ──────────────────────
88
+ by_type: dict[str, list] = {}
89
+ for atom in clean_atoms:
90
+ by_type.setdefault(atom.atom_type, []).append(atom)
91
+
92
+ for atom_type, title in self.SECTION_TITLES.items():
93
+ atoms = by_type.get(atom_type, [])
94
+ if not atoms:
95
+ continue
96
+ lines.append(title)
97
+ lines.append("")
98
+ for atom in atoms:
99
+ self._render_atom(lines, atom, bundle, video_url)
100
+ lines.append("")
101
+
102
+ # ── Workflow tips as callouts ──────────────────────────
103
+ workflow_atoms = by_type.get("workflow_step", [])
104
+ if workflow_atoms:
105
+ lines.append("## 📋 Workflow Tips")
106
+ lines.append("")
107
+ for i, atom in enumerate(workflow_atoms, 1):
108
+ content = MarkdownRenderer.clean_content(atom.content)
109
+ if not content:
110
+ continue
111
+ lines.append(f"> [!tip] Step {i}: {content}")
112
+ if atom.explanation:
113
+ lines.append(f"> {atom.explanation}")
114
+ for ts in atom.timestamps:
115
+ m, s = divmod(int(ts), 60)
116
+ lines.append(f"> [{m:02d}:{s:02d}]({video_url}&t={int(ts)}s)")
117
+ lines.append("")
118
+
119
+ output_path.write_text("\n".join(lines), encoding="utf-8")
120
+
121
+ def _render_atom(
122
+ self, lines: list[str], atom: Any, bundle: Any, video_url: str
123
+ ) -> None:
124
+ if atom.atom_type == "core_concept":
125
+ lines.append(f"- **[[{atom.content}]]** — {atom.explanation}")
126
+ elif atom.atom_type == "terminal_command":
127
+ lines.append(f"- `{atom.content}` — {atom.explanation}")
128
+ else:
129
+ lines.append(f"- {atom.content}")
130
+ if atom.explanation:
131
+ lines.append(f" > {atom.explanation}")
132
+
133
+ for ts in atom.timestamps[:3]:
134
+ m, s = divmod(int(ts), 60)
135
+ lines.append(f" — [{m:02d}:{s:02d}]({video_url}&t={int(ts)}s)")
136
+ if len(atom.timestamps) > 3:
137
+ lines.append(f" — *+{len(atom.timestamps) - 3} more timestamps*")
138
+ lines.append("")