kd-engine 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kd_engine-1.0.0.dist-info/METADATA +19 -0
- kd_engine-1.0.0.dist-info/RECORD +118 -0
- kd_engine-1.0.0.dist-info/WHEEL +5 -0
- kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
- kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
- kd_engine-1.0.0.dist-info/top_level.txt +1 -0
- ytk/__init__.py +2 -0
- ytk/__main__.py +21 -0
- ytk/batch/__init__.py +39 -0
- ytk/batch/runner.py +1149 -0
- ytk/cache/__init__.py +34 -0
- ytk/cache/v2.py +562 -0
- ytk/cleaning/density.py +40 -0
- ytk/cleaning/fillers.py +18 -0
- ytk/cleaning/sentence_boundary.py +30 -0
- ytk/cleaning/service.py +62 -0
- ytk/cli.py +1281 -0
- ytk/commands/__init__.py +3 -0
- ytk/commands/merge.py +600 -0
- ytk/config.py +757 -0
- ytk/dedup/service.py +166 -0
- ytk/doctor.py +891 -0
- ytk/dotenv.py +334 -0
- ytk/errors.py +38 -0
- ytk/extraction/service.py +451 -0
- ytk/extraction/strategies/base.py +16 -0
- ytk/extraction/strategies/code_block.py +285 -0
- ytk/extraction/strategies/heuristic.py +791 -0
- ytk/extraction/strategies/local_model.py +56 -0
- ytk/extraction/text_context.py +177 -0
- ytk/extraction/visual_context.py +159 -0
- ytk/ingestion/adapters/__init__.py +1 -0
- ytk/ingestion/adapters/_common.py +865 -0
- ytk/ingestion/adapters/base.py +160 -0
- ytk/ingestion/adapters/bilibili.py +271 -0
- ytk/ingestion/adapters/douyin.py +445 -0
- ytk/ingestion/adapters/tiktok.py +102 -0
- ytk/ingestion/adapters/xiaohongshu.py +758 -0
- ytk/ingestion/adapters/youtube.py +74 -0
- ytk/ingestion/browser_acquired.py +220 -0
- ytk/ingestion/http_client.py +43 -0
- ytk/ingestion/local_import.py +202 -0
- ytk/ingestion/media.py +468 -0
- ytk/ingestion/providers/asr.py +902 -0
- ytk/ingestion/providers/base.py +13 -0
- ytk/ingestion/providers/dtk_client.py +816 -0
- ytk/ingestion/providers/minimax.py +350 -0
- ytk/ingestion/providers/transcript_api.py +336 -0
- ytk/ingestion/providers/xhs_client.py +646 -0
- ytk/ingestion/providers/ytdlp.py +400 -0
- ytk/ingestion/registry.py +189 -0
- ytk/ingestion/resolver.py +417 -0
- ytk/ingestion/search.py +56 -0
- ytk/ingestion/service.py +167 -0
- ytk/ingestion/subtitles.py +602 -0
- ytk/ingestion/transcript_cache.py +212 -0
- ytk/ingestion/url_parser.py +53 -0
- ytk/ingestion/video_frames.py +292 -0
- ytk/ingestion/ytdlp_acquire.py +754 -0
- ytk/logging.py +383 -0
- ytk/models/common.py +20 -0
- ytk/models/content.py +451 -0
- ytk/models/knowledge.py +43 -0
- ytk/models/migrations.py +262 -0
- ytk/models/skill.py +13 -0
- ytk/models/source.py +265 -0
- ytk/models/topic.py +18 -0
- ytk/models/transcript.py +30 -0
- ytk/ocr/__init__.py +65 -0
- ytk/ocr/engine.py +245 -0
- ytk/ocr/layout.py +347 -0
- ytk/ocr/pipeline.py +440 -0
- ytk/ocr/rapidocr_engine.py +144 -0
- ytk/pipeline/acquisition_checkpoint.py +240 -0
- ytk/pipeline/artifacts.py +18 -0
- ytk/pipeline/orchestrator.py +598 -0
- ytk/pipeline/processing_identity.py +18 -0
- ytk/pipeline/retry.py +728 -0
- ytk/pipeline/runners.py +578 -0
- ytk/pipeline/state.py +1107 -0
- ytk/pipeline/v2.py +1863 -0
- ytk/py.typed +1 -0
- ytk/quality/__init__.py +9 -0
- ytk/quality/completeness.py +260 -0
- ytk/renderers/base.py +9 -0
- ytk/renderers/citation_alignment.py +617 -0
- ytk/renderers/evidence_formatter.py +362 -0
- ytk/renderers/json_renderer.py +13 -0
- ytk/renderers/jsonl_renderer.py +18 -0
- ytk/renderers/knowledge_doc_renderer.py +1984 -0
- ytk/renderers/markdown_renderer.py +540 -0
- ytk/renderers/obsidian_renderer.py +138 -0
- ytk/renderers/skill_package.py +21 -0
- ytk/renderers/source_links.py +309 -0
- ytk/renderers/summary_renderer.py +163 -0
- ytk/renderers/v2.py +171 -0
- ytk/renderers/v2_jsonl.py +165 -0
- ytk/renderers/v2_markdown.py +477 -0
- ytk/renderers/v2_obsidian.py +113 -0
- ytk/renderers/v2_summary.py +220 -0
- ytk/segmentation/change_points.py +50 -0
- ytk/segmentation/embeddings.py +45 -0
- ytk/segmentation/service.py +318 -0
- ytk/skill_builder/__init__.py +1 -0
- ytk/skill_builder/builder.py +419 -0
- ytk/skill_builder/templates/SKILL.md.j2 +85 -0
- ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
- ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
- ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
- ytk/skill_builder/templates/references/commands.md.j2 +34 -0
- ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
- ytk/skill_builder/templates/references/configs.md.j2 +31 -0
- ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
- ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
- ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
- ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
- ytk/skill_builder/templates.py +29 -0
- ytk/text_normalization.py +44 -0
|
@@ -0,0 +1,540 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from collections import defaultdict
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def source_processing_notice(bundle: Any) -> str:
|
|
10
|
+
"""Keep acquisition provenance distinct from human review and visual coverage."""
|
|
11
|
+
source = getattr(bundle, "transcript_source", "unknown")
|
|
12
|
+
notice = {
|
|
13
|
+
"manual": "Extracted from manual captions",
|
|
14
|
+
"auto": "Extracted from auto-generated captions; not human reviewed",
|
|
15
|
+
"asr": "Extracted from ASR speech-to-text; not human reviewed",
|
|
16
|
+
"yt_dlp": "Transcript acquired via yt-dlp; caption type not independently verified",
|
|
17
|
+
"minimax": "Cloud-model video interpretation; not human reviewed",
|
|
18
|
+
}.get(source, "Transcript source unknown")
|
|
19
|
+
scope = ("Visual coverage not independently verified (画面覆盖范围未独立验证)"
|
|
20
|
+
if source == "minimax" else "Visuals not analysed (未分析画面); slides and demonstrations may be missing")
|
|
21
|
+
return notice + ". " + scope + "."
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class MarkdownRenderer:
|
|
25
|
+
"""Render an ExtractionBundle into a clean, human-readable Markdown note."""
|
|
26
|
+
|
|
27
|
+
# ── Garbage-detection patterns ──────────────────────────────────
|
|
28
|
+
_TRUNCATION_ENDINGS = re.compile(
|
|
29
|
+
r"(?:In|The|Once|Now we are|or|and|the|a|in|But|So|That|This|It|We|They"
|
|
30
|
+
r"|You|He|She|What|How|Why|When|Where|Who|Is|Are|Was|Were|Do|Does|Did"
|
|
31
|
+
r"|Can|Could|Will|Would|Should|May|Might|Must|Shall|Has|Have|Had"
|
|
32
|
+
r"|Not|No|Yes|But|Just|Only|Also|Even|Still|Already|Yet|Now|Then"
|
|
33
|
+
r"|Here|There|Well|OK|Okay|Right|Sure|Good|Great|Fine|Nice"
|
|
34
|
+
r"|Like|Just|Really|Very|Much|More|Most|Less|Least|Too|Also)\.$"
|
|
35
|
+
)
|
|
36
|
+
_NATURAL_LANGUAGE_FRAGMENT = re.compile(
|
|
37
|
+
r"^[A-Z][a-z].*\b(?:can|will|is|are|was|were|it|we|you|they|the|this|that)\b"
|
|
38
|
+
)
|
|
39
|
+
_YAML_SPECIAL_RE = re.compile(r"[:#&*!|>%@`]")
|
|
40
|
+
|
|
41
|
+
# ── Public API ─────────────────────────────────────────────────
|
|
42
|
+
|
|
43
|
+
def render(self, bundle: Any, output_path: Path, **kwargs: Any) -> None:
|
|
44
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
45
|
+
|
|
46
|
+
video_url = bundle.metadata.get(
|
|
47
|
+
"canonical_url",
|
|
48
|
+
f"https://www.youtube.com/watch?v={bundle.video_id}",
|
|
49
|
+
)
|
|
50
|
+
title = str(bundle.metadata.get("title", bundle.video_id))
|
|
51
|
+
|
|
52
|
+
src = bundle.transcript_source
|
|
53
|
+
clean_atoms = [a for a in bundle.knowledge_atoms if not self._is_garbage(a, source=src)]
|
|
54
|
+
|
|
55
|
+
lines: list[str] = []
|
|
56
|
+
|
|
57
|
+
# ── Header ─────────────────────────────────────────────
|
|
58
|
+
lines.append(f"# {title}")
|
|
59
|
+
lines.append("")
|
|
60
|
+
lines.append(f"> 📺 [Watch on YouTube]({video_url})")
|
|
61
|
+
caption_note = source_processing_notice(bundle)
|
|
62
|
+
lines.append(f"> 🕐 {caption_note}")
|
|
63
|
+
lines.append("")
|
|
64
|
+
lines.append("---")
|
|
65
|
+
lines.append("")
|
|
66
|
+
|
|
67
|
+
# ── Summary (legacy section — tests assert exact format) ──
|
|
68
|
+
lines.append("## Summary")
|
|
69
|
+
lines.append("")
|
|
70
|
+
lines.append(f"- Video ID: `{bundle.video_id}`")
|
|
71
|
+
lines.append(f"- Transcript source: `{bundle.transcript_source}`")
|
|
72
|
+
|
|
73
|
+
canonical_url = bundle.metadata.get("canonical_url")
|
|
74
|
+
if canonical_url:
|
|
75
|
+
lines.append(f"- Canonical URL: {canonical_url}")
|
|
76
|
+
|
|
77
|
+
lines.append("")
|
|
78
|
+
|
|
79
|
+
# ── Overview (new beautiful section) ───────────────────
|
|
80
|
+
overview = self._build_overview(bundle, clean_atoms)
|
|
81
|
+
if overview:
|
|
82
|
+
lines.append("## Overview")
|
|
83
|
+
lines.append("")
|
|
84
|
+
for para in overview:
|
|
85
|
+
lines.append(para)
|
|
86
|
+
lines.append("")
|
|
87
|
+
lines.append("---")
|
|
88
|
+
lines.append("")
|
|
89
|
+
|
|
90
|
+
# ── Executive Summary (legacy — tests assert "stands out in") ──
|
|
91
|
+
executive_summary = self._build_executive_summary(bundle, clean_atoms)
|
|
92
|
+
if executive_summary:
|
|
93
|
+
lines.append("## Executive Summary")
|
|
94
|
+
lines.append("")
|
|
95
|
+
for line in executive_summary:
|
|
96
|
+
lines.append(f"- {line}")
|
|
97
|
+
lines.append("")
|
|
98
|
+
|
|
99
|
+
# ── Key Takeaways (legacy — tests assert this structure) ──
|
|
100
|
+
grouped = self._group_atoms(clean_atoms)
|
|
101
|
+
if any(grouped.values()):
|
|
102
|
+
lines.append("## Key Takeaways")
|
|
103
|
+
lines.append("")
|
|
104
|
+
self._append_group(
|
|
105
|
+
lines,
|
|
106
|
+
"Commands",
|
|
107
|
+
grouped["terminal_command"],
|
|
108
|
+
formatter=lambda atom: f"- `{atom.content}`",
|
|
109
|
+
)
|
|
110
|
+
self._append_group(
|
|
111
|
+
lines,
|
|
112
|
+
"Files & Config",
|
|
113
|
+
grouped["configuration_snippet"],
|
|
114
|
+
formatter=lambda atom: f"- `{atom.content}`",
|
|
115
|
+
)
|
|
116
|
+
self._append_group(
|
|
117
|
+
lines,
|
|
118
|
+
"Concepts",
|
|
119
|
+
grouped["core_concept"],
|
|
120
|
+
formatter=lambda atom: f"- `{atom.content}`: {self._display_explanation(atom)}",
|
|
121
|
+
)
|
|
122
|
+
self._append_group(
|
|
123
|
+
lines,
|
|
124
|
+
"Workflows",
|
|
125
|
+
grouped["workflow_step"] + grouped["best_practice"],
|
|
126
|
+
formatter=lambda atom: f"- {atom.content}",
|
|
127
|
+
)
|
|
128
|
+
self._append_group(
|
|
129
|
+
lines,
|
|
130
|
+
"Code Blocks",
|
|
131
|
+
grouped["code_block"],
|
|
132
|
+
formatter=lambda atom: f"- `{atom.content}`",
|
|
133
|
+
)
|
|
134
|
+
self._append_group(
|
|
135
|
+
lines,
|
|
136
|
+
"Command Sequences",
|
|
137
|
+
grouped["command_sequence"],
|
|
138
|
+
formatter=lambda atom: f"- `{atom.content}`",
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
# ── Topic Notes (legacy — tests assert atom details) ───
|
|
142
|
+
lines.append("## Topic Notes")
|
|
143
|
+
lines.append("")
|
|
144
|
+
|
|
145
|
+
atoms_by_topic = defaultdict(list)
|
|
146
|
+
for atom in clean_atoms:
|
|
147
|
+
topic_key = self._topic_key_from_path(atom.topic_path)
|
|
148
|
+
atoms_by_topic[topic_key].append(atom)
|
|
149
|
+
|
|
150
|
+
rendered_topics: set[str] = set()
|
|
151
|
+
for topic in bundle.topic_segments:
|
|
152
|
+
rendered_topics.add(topic.id)
|
|
153
|
+
lines.append(f"## {topic.title} ({topic.start:.1f}s–{topic.end:.1f}s)")
|
|
154
|
+
lines.append("")
|
|
155
|
+
if topic.sentences:
|
|
156
|
+
lines.append(f"- Summary: {topic.sentences[0]}")
|
|
157
|
+
lines.append("")
|
|
158
|
+
|
|
159
|
+
for atom in atoms_by_topic.get(topic.id, []):
|
|
160
|
+
lines.append(
|
|
161
|
+
f"### {atom.atom_type.replace('_', ' ').title()}: {atom.content}"
|
|
162
|
+
)
|
|
163
|
+
lines.append(f"- Confidence: {atom.confidence:.2f}")
|
|
164
|
+
lines.append(f"- Explanation: {atom.explanation}")
|
|
165
|
+
for evidence in atom.evidence:
|
|
166
|
+
lines.append(
|
|
167
|
+
f"- Evidence: `{evidence.text}` "
|
|
168
|
+
f"({evidence.start:.1f}s-{evidence.end:.1f}s, {evidence.source})"
|
|
169
|
+
)
|
|
170
|
+
for timestamp in atom.timestamps:
|
|
171
|
+
lines.append(
|
|
172
|
+
f"- Watch: https://www.youtube.com/watch"
|
|
173
|
+
f"?v={bundle.video_id}&t={int(timestamp)}s"
|
|
174
|
+
)
|
|
175
|
+
lines.append("")
|
|
176
|
+
|
|
177
|
+
for topic_key, atoms in atoms_by_topic.items():
|
|
178
|
+
if topic_key in rendered_topics:
|
|
179
|
+
continue
|
|
180
|
+
display_name = (
|
|
181
|
+
atoms[0].topic_path[0] if atoms and atoms[0].topic_path else "Ungrouped"
|
|
182
|
+
)
|
|
183
|
+
lines.append(f"## {display_name}")
|
|
184
|
+
lines.append("")
|
|
185
|
+
for atom in atoms:
|
|
186
|
+
lines.append(
|
|
187
|
+
f"### {atom.atom_type.replace('_', ' ').title()}: {atom.content}"
|
|
188
|
+
)
|
|
189
|
+
lines.append(f"- Confidence: {atom.confidence:.2f}")
|
|
190
|
+
lines.append(f"- Explanation: {atom.explanation}")
|
|
191
|
+
for evidence in atom.evidence:
|
|
192
|
+
lines.append(
|
|
193
|
+
f"- Evidence: `{evidence.text}` "
|
|
194
|
+
f"({evidence.start:.1f}s-{evidence.end:.1f}s, {evidence.source})"
|
|
195
|
+
)
|
|
196
|
+
for timestamp in atom.timestamps:
|
|
197
|
+
lines.append(
|
|
198
|
+
f"- Watch: https://www.youtube.com/watch"
|
|
199
|
+
f"?v={bundle.video_id}&t={int(timestamp)}s"
|
|
200
|
+
)
|
|
201
|
+
lines.append("")
|
|
202
|
+
|
|
203
|
+
# ── Stats footer ───────────────────────────────────────
|
|
204
|
+
lines.append("")
|
|
205
|
+
lines.append("---")
|
|
206
|
+
lines.append("")
|
|
207
|
+
lines.append("## Stats")
|
|
208
|
+
lines.append("")
|
|
209
|
+
lines.append(f"- Total knowledge atoms: {len(bundle.knowledge_atoms)}")
|
|
210
|
+
type_counts: dict[str, int] = {}
|
|
211
|
+
for atom in bundle.knowledge_atoms:
|
|
212
|
+
type_counts[atom.atom_type] = type_counts.get(atom.atom_type, 0) + 1
|
|
213
|
+
for atype, count in sorted(type_counts.items()):
|
|
214
|
+
lines.append(f"- {atype.replace('_', ' ').title()}: {count}")
|
|
215
|
+
|
|
216
|
+
lines.append("")
|
|
217
|
+
lines.append(
|
|
218
|
+
f"*Generated by knowledge-distiller v2.0"
|
|
219
|
+
f" • {len(bundle.knowledge_atoms)} knowledge atoms extracted*"
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
output_path.write_text("\n".join(lines), encoding="utf-8")
|
|
223
|
+
|
|
224
|
+
# ── Sanitization helpers ───────────────────────────────────────
|
|
225
|
+
|
|
226
|
+
@classmethod
|
|
227
|
+
def _is_garbage(cls, atom: Any, *, source: str = "auto") -> bool:
|
|
228
|
+
"""Return True when an atom is noise that shouldn't appear in output.
|
|
229
|
+
|
|
230
|
+
Universal checks (all sources):
|
|
231
|
+
- Very low confidence
|
|
232
|
+
- && join artifacts from heuristic extractor
|
|
233
|
+
- Terminal commands with slash prefix + boilerplate explanation
|
|
234
|
+
- Workflow steps with heuristic template explanation
|
|
235
|
+
- Core concepts with truncated descriptions
|
|
236
|
+
|
|
237
|
+
Auto/ASR only checks:
|
|
238
|
+
- Mid-sentence fragments (lowercase start)
|
|
239
|
+
- Too few words
|
|
240
|
+
- Truncated caption endings
|
|
241
|
+
- Natural language command_sequence fragments
|
|
242
|
+
|
|
243
|
+
Note on "template" rules: an atom with a heuristic-style template
|
|
244
|
+
explanation (e.g. "This sentence expresses an ordered workflow
|
|
245
|
+
instruction.") is only treated as noise when it has no transcript
|
|
246
|
+
evidence attached. Real tutorial extractions always carry evidence
|
|
247
|
+
with a start/end timestamp pointing back to the source caption, so
|
|
248
|
+
keeping those atoms preserves the most useful tutorial knowledge
|
|
249
|
+
rather than silently dropping it. v2.0 originally filtered these
|
|
250
|
+
unconditionally; that turned out to discard the bulk of the
|
|
251
|
+
workflow_step and command_sequence atoms from real YouTube
|
|
252
|
+
tutorial transcripts (validated on vRYBG_R8JAI, 2026-06-07).
|
|
253
|
+
"""
|
|
254
|
+
content = str(atom.content).strip()
|
|
255
|
+
explanation = str(getattr(atom, "explanation", "")).strip()
|
|
256
|
+
has_evidence = bool(getattr(atom, "evidence", None))
|
|
257
|
+
|
|
258
|
+
# ── Universal garbage patterns ──────────────────────────
|
|
259
|
+
|
|
260
|
+
# Confidence too low
|
|
261
|
+
if atom.confidence < 0.5:
|
|
262
|
+
return True
|
|
263
|
+
|
|
264
|
+
# Join artifacts from heuristic extractor (always noise)
|
|
265
|
+
# KEEP if the atom has transcript evidence — heuristic joins are
|
|
266
|
+
# noise when fabricating, but legitimate when each step is grounded
|
|
267
|
+
# in the source transcript.
|
|
268
|
+
if "&&" in content and not has_evidence:
|
|
269
|
+
return True
|
|
270
|
+
|
|
271
|
+
# Heuristic template workflow steps (not real instructions)
|
|
272
|
+
# KEEP if the atom has transcript evidence — same reasoning.
|
|
273
|
+
if (
|
|
274
|
+
atom.atom_type == "workflow_step"
|
|
275
|
+
and explanation == "This sentence expresses an ordered workflow instruction."
|
|
276
|
+
and not has_evidence
|
|
277
|
+
):
|
|
278
|
+
return True
|
|
279
|
+
|
|
280
|
+
# Core concept with truncated (very short) description
|
|
281
|
+
grounded_chinese = has_evidence and len(re.findall(r"[\u4e00-\u9fff]", content + explanation)) >= 4
|
|
282
|
+
if atom.atom_type == "core_concept" and len(explanation) < 10 and not grounded_chinese:
|
|
283
|
+
return True
|
|
284
|
+
|
|
285
|
+
# Heuristic template core concept (a/an X is a Y) — keep if
|
|
286
|
+
# evidence is present so we surface real tutorial concepts.
|
|
287
|
+
if (
|
|
288
|
+
atom.atom_type == "core_concept"
|
|
289
|
+
and "mentioned in the video that relates to" in explanation
|
|
290
|
+
and not has_evidence
|
|
291
|
+
):
|
|
292
|
+
return True
|
|
293
|
+
|
|
294
|
+
# ── Auto/ASR only weak-signal patterns ──────────────────
|
|
295
|
+
if source not in ("auto", "asr"):
|
|
296
|
+
return False
|
|
297
|
+
|
|
298
|
+
# Chinese clauses do not use spaces as word boundaries. English word
|
|
299
|
+
# counts otherwise discard every grounded Chinese ASR instruction.
|
|
300
|
+
if grounded_chinese:
|
|
301
|
+
return False
|
|
302
|
+
|
|
303
|
+
# Terminal commands that are auto-caption truncations
|
|
304
|
+
if (
|
|
305
|
+
atom.atom_type == "terminal_command"
|
|
306
|
+
and content.startswith("/")
|
|
307
|
+
and "as shown in the transcript" in explanation
|
|
308
|
+
):
|
|
309
|
+
return True
|
|
310
|
+
|
|
311
|
+
# Core concept with template-generated explanation (auto captions only)
|
|
312
|
+
if atom.atom_type == "core_concept" and (
|
|
313
|
+
explanation
|
|
314
|
+
== "Important Claude Code workflow concept mentioned in the transcript."
|
|
315
|
+
):
|
|
316
|
+
return True
|
|
317
|
+
if content and content[0].islower() and atom.atom_type != "terminal_command":
|
|
318
|
+
return True
|
|
319
|
+
|
|
320
|
+
# Too short and not a terminal command
|
|
321
|
+
word_count = len(content.split())
|
|
322
|
+
if word_count < 5 and atom.atom_type != "terminal_command":
|
|
323
|
+
return True
|
|
324
|
+
|
|
325
|
+
# Truncated auto-caption endings
|
|
326
|
+
if cls._TRUNCATION_ENDINGS.search(content):
|
|
327
|
+
return True
|
|
328
|
+
|
|
329
|
+
# Command sequence atoms that are natural language, not commands
|
|
330
|
+
if atom.atom_type == "command_sequence" and cls._NATURAL_LANGUAGE_FRAGMENT.match(
|
|
331
|
+
content
|
|
332
|
+
):
|
|
333
|
+
return True
|
|
334
|
+
|
|
335
|
+
return False
|
|
336
|
+
|
|
337
|
+
@classmethod
|
|
338
|
+
def clean_content(cls, text: str) -> str:
|
|
339
|
+
"""Sanitize a single piece of content text."""
|
|
340
|
+
text = text.strip()
|
|
341
|
+
text = re.sub(r"\s*(?:stands out|is used|is covered|is discussed)\s+in\s+.*$", "", text)
|
|
342
|
+
if text and text[0].islower():
|
|
343
|
+
text = text[0].upper() + text[1:]
|
|
344
|
+
return text
|
|
345
|
+
|
|
346
|
+
@classmethod
|
|
347
|
+
def clean_tag(cls, tag: str) -> str:
|
|
348
|
+
"""Produce a clean Obsidian-safe tag."""
|
|
349
|
+
tag = tag.strip()
|
|
350
|
+
tag = tag.replace("\n", " ").replace("\r", " ")
|
|
351
|
+
tag = re.sub(r"[.,;!?()\s]+$", "", tag)
|
|
352
|
+
tag = re.sub(r"^[.,;!?()\s]+", "", tag)
|
|
353
|
+
tag = cls._YAML_SPECIAL_RE.sub("", tag)
|
|
354
|
+
tag = tag.replace(",", "-")
|
|
355
|
+
tag = tag.replace(" ", "-")
|
|
356
|
+
tag = re.sub(r"-+", "-", tag)
|
|
357
|
+
tag = tag.strip("-")
|
|
358
|
+
if re.match(r"^topic-\d+$", tag):
|
|
359
|
+
return ""
|
|
360
|
+
if len(tag) < 3:
|
|
361
|
+
return ""
|
|
362
|
+
if len(tag.split("-")) >= 5:
|
|
363
|
+
return ""
|
|
364
|
+
return tag
|
|
365
|
+
|
|
366
|
+
# ── Overview builder ───────────────────────────────────────────
|
|
367
|
+
|
|
368
|
+
def _build_overview(self, bundle: Any, clean_atoms: list) -> list[str]:
|
|
369
|
+
"""Build 2-3 sentence overview from topic segments."""
|
|
370
|
+
paragraphs: list[str] = []
|
|
371
|
+
sentences: list[str] = []
|
|
372
|
+
for topic in bundle.topic_segments:
|
|
373
|
+
for sent in topic.sentences:
|
|
374
|
+
cleaned = self.clean_content(sent)
|
|
375
|
+
if cleaned and len(cleaned.split()) >= 5:
|
|
376
|
+
sentences.append(cleaned)
|
|
377
|
+
if len(sentences) >= 3:
|
|
378
|
+
break
|
|
379
|
+
if len(sentences) >= 3:
|
|
380
|
+
break
|
|
381
|
+
if sentences:
|
|
382
|
+
paragraphs.append(" ".join(sentences))
|
|
383
|
+
return paragraphs
|
|
384
|
+
|
|
385
|
+
if clean_atoms:
|
|
386
|
+
ranked = sorted(clean_atoms, key=lambda a: a.confidence, reverse=True)
|
|
387
|
+
overview_parts = [
|
|
388
|
+
self.clean_content(a.content) for a in ranked[:3] if self.clean_content(a.content)
|
|
389
|
+
]
|
|
390
|
+
if overview_parts:
|
|
391
|
+
paragraphs.append(" ".join(overview_parts))
|
|
392
|
+
return paragraphs
|
|
393
|
+
|
|
394
|
+
return ["No reliable overview could be extracted from this video's captions."]
|
|
395
|
+
|
|
396
|
+
# ── Executive Summary (keeps "stands out in" for tests) ────────
|
|
397
|
+
|
|
398
|
+
def _build_executive_summary(self, bundle: Any, clean_atoms: list) -> list[str]:
|
|
399
|
+
atoms_by_topic: dict[str, list] = defaultdict(list)
|
|
400
|
+
for atom in clean_atoms:
|
|
401
|
+
topic_key = self._topic_key_from_path(atom.topic_path)
|
|
402
|
+
atoms_by_topic[topic_key].append(atom)
|
|
403
|
+
|
|
404
|
+
summary_lines: list[str] = []
|
|
405
|
+
topic_order = [(t.id, t.title) for t in bundle.topic_segments]
|
|
406
|
+
for topic_key, topic_name in topic_order:
|
|
407
|
+
atoms = atoms_by_topic.get(topic_key, [])
|
|
408
|
+
if not atoms:
|
|
409
|
+
continue
|
|
410
|
+
ranked = sorted(
|
|
411
|
+
atoms,
|
|
412
|
+
key=lambda a: (
|
|
413
|
+
self._atom_priority(a.atom_type),
|
|
414
|
+
self._atom_specificity(a),
|
|
415
|
+
self._atom_summary_rank(a),
|
|
416
|
+
a.confidence,
|
|
417
|
+
),
|
|
418
|
+
reverse=True,
|
|
419
|
+
)
|
|
420
|
+
if self._topic_is_generic_only(ranked):
|
|
421
|
+
continue
|
|
422
|
+
used: set[str] = set()
|
|
423
|
+
max_items = 2 if self._topic_supports_dense_summary(ranked) else 1
|
|
424
|
+
added = 0
|
|
425
|
+
for atom in ranked:
|
|
426
|
+
key = str(atom.content).casefold()
|
|
427
|
+
if key in used:
|
|
428
|
+
continue
|
|
429
|
+
if self._should_skip_summary_atom(atom, ranked):
|
|
430
|
+
continue
|
|
431
|
+
used.add(key)
|
|
432
|
+
summary_lines.append(self._summary_sentence(topic_name, atom))
|
|
433
|
+
added += 1
|
|
434
|
+
if len(summary_lines) >= 5 or added >= max_items:
|
|
435
|
+
break
|
|
436
|
+
if len(summary_lines) >= 5:
|
|
437
|
+
break
|
|
438
|
+
return summary_lines
|
|
439
|
+
|
|
440
|
+
# ── Legacy helpers (kept for test compatibility) ───────────────
|
|
441
|
+
|
|
442
|
+
@staticmethod
|
|
443
|
+
def _topic_key_from_path(topic_path: list) -> str:
|
|
444
|
+
if len(topic_path) >= 2:
|
|
445
|
+
return str(topic_path[1])
|
|
446
|
+
if topic_path:
|
|
447
|
+
return str(topic_path[0])
|
|
448
|
+
return "Ungrouped"
|
|
449
|
+
|
|
450
|
+
@staticmethod
|
|
451
|
+
def _atom_priority(atom_type: str) -> int:
|
|
452
|
+
return {
|
|
453
|
+
"terminal_command": 5,
|
|
454
|
+
"configuration_snippet": 4,
|
|
455
|
+
"core_concept": 3,
|
|
456
|
+
"best_practice": 2,
|
|
457
|
+
"workflow_step": 1,
|
|
458
|
+
"code_block": 4,
|
|
459
|
+
"command_sequence": 5,
|
|
460
|
+
}.get(atom_type, 0)
|
|
461
|
+
|
|
462
|
+
@staticmethod
|
|
463
|
+
def _atom_specificity(atom: Any) -> int:
|
|
464
|
+
return len(str(atom.content).split())
|
|
465
|
+
|
|
466
|
+
@staticmethod
|
|
467
|
+
def _atom_summary_rank(atom: Any) -> int:
|
|
468
|
+
generic = {"claude code", "terminal", "pattern", "patterns"}
|
|
469
|
+
if atom.atom_type == "core_concept" and str(atom.content).casefold() in generic:
|
|
470
|
+
return 0
|
|
471
|
+
return 1
|
|
472
|
+
|
|
473
|
+
def _topic_supports_dense_summary(self, ranked: list) -> bool:
|
|
474
|
+
specific = [
|
|
475
|
+
a for a in ranked if a.atom_type == "core_concept" and self._atom_summary_rank(a) > 0
|
|
476
|
+
]
|
|
477
|
+
return len(specific) >= 2
|
|
478
|
+
|
|
479
|
+
def _topic_is_generic_only(self, ranked: list) -> bool:
|
|
480
|
+
if not ranked:
|
|
481
|
+
return True
|
|
482
|
+
return all(
|
|
483
|
+
a.atom_type == "core_concept" and self._atom_summary_rank(a) == 0 for a in ranked
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
def _should_skip_summary_atom(self, atom: Any, ranked: list) -> bool:
|
|
487
|
+
if atom.atom_type != "core_concept":
|
|
488
|
+
return False
|
|
489
|
+
if self._atom_summary_rank(atom) > 0:
|
|
490
|
+
return False
|
|
491
|
+
return any(
|
|
492
|
+
o.atom_type == "core_concept" and self._atom_summary_rank(o) > 0 for o in ranked
|
|
493
|
+
)
|
|
494
|
+
|
|
495
|
+
def _summary_sentence(self, topic_name: str, atom: Any) -> str:
|
|
496
|
+
if atom.atom_type in {"terminal_command", "configuration_snippet"}:
|
|
497
|
+
return f"`{atom.content}` is used in {topic_name}."
|
|
498
|
+
if atom.atom_type == "core_concept":
|
|
499
|
+
return f"{atom.content} is covered in {topic_name}."
|
|
500
|
+
return f"{atom.content} is discussed in {topic_name}."
|
|
501
|
+
|
|
502
|
+
@staticmethod
|
|
503
|
+
def _display_explanation(atom: Any) -> str:
|
|
504
|
+
if (
|
|
505
|
+
atom.atom_type == "core_concept"
|
|
506
|
+
and atom.explanation
|
|
507
|
+
== "Important Claude Code workflow concept mentioned in the transcript."
|
|
508
|
+
):
|
|
509
|
+
return "highlighted as an important Claude Code concept."
|
|
510
|
+
return atom.explanation
|
|
511
|
+
|
|
512
|
+
@staticmethod
|
|
513
|
+
def _group_atoms(atoms: list) -> dict[str, list]:
|
|
514
|
+
types = [
|
|
515
|
+
"terminal_command",
|
|
516
|
+
"configuration_snippet",
|
|
517
|
+
"core_concept",
|
|
518
|
+
"workflow_step",
|
|
519
|
+
"best_practice",
|
|
520
|
+
"code_block",
|
|
521
|
+
"command_sequence",
|
|
522
|
+
]
|
|
523
|
+
return {t: [a for a in atoms if a.atom_type == t] for t in types}
|
|
524
|
+
|
|
525
|
+
@staticmethod
|
|
526
|
+
def _append_group(
|
|
527
|
+
lines: list[str], title: str, atoms: list, *, formatter: Any
|
|
528
|
+
) -> None:
|
|
529
|
+
if not atoms:
|
|
530
|
+
return
|
|
531
|
+
lines.append(f"## {title}")
|
|
532
|
+
lines.append("")
|
|
533
|
+
seen: set[str] = set()
|
|
534
|
+
for atom in atoms:
|
|
535
|
+
rendered = formatter(atom)
|
|
536
|
+
if rendered in seen:
|
|
537
|
+
continue
|
|
538
|
+
seen.add(rendered)
|
|
539
|
+
lines.append(rendered)
|
|
540
|
+
lines.append("")
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from ytk.renderers.markdown_renderer import MarkdownRenderer
|
|
8
|
+
|
|
9
|
+
_YAML_SPECIAL_RE = re.compile(r"[:#&*!|>%@`]")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _escape_yaml_string(value: str) -> str:
|
|
13
|
+
escaped = value.replace("\\", "\\\\").replace('"', '\\"')
|
|
14
|
+
escaped = escaped.replace("\n", " ")
|
|
15
|
+
return escaped
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class ObsidianRenderer:
|
|
19
|
+
"""Render an ExtractionBundle as an Obsidian-flavoured Markdown note."""
|
|
20
|
+
|
|
21
|
+
SECTION_TITLES: dict[str, str] = {
|
|
22
|
+
"terminal_command": "## ⌨️ Commands",
|
|
23
|
+
"best_practice": "## ✅ Best Practices",
|
|
24
|
+
"configuration_snippet": "## ⚙️ Configuration",
|
|
25
|
+
"workflow_step": "## 📋 Workflow Steps",
|
|
26
|
+
"core_concept": "## 💡 Core Concepts",
|
|
27
|
+
"code_block": "## 📦 Code Blocks",
|
|
28
|
+
"command_sequence": "## 🔗 Command Sequences",
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
def render(self, bundle: Any, output_path: Path, **kwargs: Any) -> None:
|
|
32
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
33
|
+
|
|
34
|
+
video_url = bundle.metadata.get(
|
|
35
|
+
"canonical_url",
|
|
36
|
+
f"https://www.youtube.com/watch?v={bundle.video_id}",
|
|
37
|
+
)
|
|
38
|
+
raw_title = bundle.metadata.get("title", bundle.video_id)
|
|
39
|
+
safe_title = _escape_yaml_string(str(raw_title))
|
|
40
|
+
|
|
41
|
+
clean_atoms = [
|
|
42
|
+
a
|
|
43
|
+
for a in bundle.knowledge_atoms
|
|
44
|
+
if not MarkdownRenderer._is_garbage(a, source=bundle.transcript_source)
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
lines: list[str] = []
|
|
48
|
+
|
|
49
|
+
# ── YAML frontmatter ───────────────────────────────────
|
|
50
|
+
lines.append("---")
|
|
51
|
+
lines.append(f'title: "{safe_title}"')
|
|
52
|
+
lines.append(f"video_id: {bundle.video_id}")
|
|
53
|
+
if bundle.metadata.get("canonical_url"):
|
|
54
|
+
lines.append(f"url: {bundle.metadata['canonical_url']}")
|
|
55
|
+
|
|
56
|
+
all_tags: set[str] = set()
|
|
57
|
+
for atom in clean_atoms:
|
|
58
|
+
for topic in atom.topic_path:
|
|
59
|
+
tag = MarkdownRenderer.clean_tag(topic)
|
|
60
|
+
if tag:
|
|
61
|
+
all_tags.add(tag)
|
|
62
|
+
if all_tags:
|
|
63
|
+
lines.append("tags:")
|
|
64
|
+
for tag in sorted(all_tags):
|
|
65
|
+
lines.append(f" - {tag}")
|
|
66
|
+
lines.append("---")
|
|
67
|
+
lines.append("")
|
|
68
|
+
|
|
69
|
+
# ── Header ─────────────────────────────────────────────
|
|
70
|
+
lines.append(f"# {raw_title}")
|
|
71
|
+
lines.append("")
|
|
72
|
+
lines.append(f"> 📺 [Watch on YouTube]({video_url})")
|
|
73
|
+
caption_note = (
|
|
74
|
+
"Extracted from auto-generated captions"
|
|
75
|
+
if bundle.transcript_source in ("auto", "asr")
|
|
76
|
+
else "Extracted from manual captions"
|
|
77
|
+
)
|
|
78
|
+
lines.append(f"> 🕐 {caption_note}")
|
|
79
|
+
lines.append("")
|
|
80
|
+
lines.append("---")
|
|
81
|
+
lines.append("")
|
|
82
|
+
|
|
83
|
+
if not clean_atoms:
|
|
84
|
+
lines.append("> ⚠️ No reliable knowledge atoms could be extracted from this video's captions.")
|
|
85
|
+
lines.append("")
|
|
86
|
+
|
|
87
|
+
# ── Grouped sections by atom type ──────────────────────
|
|
88
|
+
by_type: dict[str, list] = {}
|
|
89
|
+
for atom in clean_atoms:
|
|
90
|
+
by_type.setdefault(atom.atom_type, []).append(atom)
|
|
91
|
+
|
|
92
|
+
for atom_type, title in self.SECTION_TITLES.items():
|
|
93
|
+
atoms = by_type.get(atom_type, [])
|
|
94
|
+
if not atoms:
|
|
95
|
+
continue
|
|
96
|
+
lines.append(title)
|
|
97
|
+
lines.append("")
|
|
98
|
+
for atom in atoms:
|
|
99
|
+
self._render_atom(lines, atom, bundle, video_url)
|
|
100
|
+
lines.append("")
|
|
101
|
+
|
|
102
|
+
# ── Workflow tips as callouts ──────────────────────────
|
|
103
|
+
workflow_atoms = by_type.get("workflow_step", [])
|
|
104
|
+
if workflow_atoms:
|
|
105
|
+
lines.append("## 📋 Workflow Tips")
|
|
106
|
+
lines.append("")
|
|
107
|
+
for i, atom in enumerate(workflow_atoms, 1):
|
|
108
|
+
content = MarkdownRenderer.clean_content(atom.content)
|
|
109
|
+
if not content:
|
|
110
|
+
continue
|
|
111
|
+
lines.append(f"> [!tip] Step {i}: {content}")
|
|
112
|
+
if atom.explanation:
|
|
113
|
+
lines.append(f"> {atom.explanation}")
|
|
114
|
+
for ts in atom.timestamps:
|
|
115
|
+
m, s = divmod(int(ts), 60)
|
|
116
|
+
lines.append(f"> [{m:02d}:{s:02d}]({video_url}&t={int(ts)}s)")
|
|
117
|
+
lines.append("")
|
|
118
|
+
|
|
119
|
+
output_path.write_text("\n".join(lines), encoding="utf-8")
|
|
120
|
+
|
|
121
|
+
def _render_atom(
|
|
122
|
+
self, lines: list[str], atom: Any, bundle: Any, video_url: str
|
|
123
|
+
) -> None:
|
|
124
|
+
if atom.atom_type == "core_concept":
|
|
125
|
+
lines.append(f"- **[[{atom.content}]]** — {atom.explanation}")
|
|
126
|
+
elif atom.atom_type == "terminal_command":
|
|
127
|
+
lines.append(f"- `{atom.content}` — {atom.explanation}")
|
|
128
|
+
else:
|
|
129
|
+
lines.append(f"- {atom.content}")
|
|
130
|
+
if atom.explanation:
|
|
131
|
+
lines.append(f" > {atom.explanation}")
|
|
132
|
+
|
|
133
|
+
for ts in atom.timestamps[:3]:
|
|
134
|
+
m, s = divmod(int(ts), 60)
|
|
135
|
+
lines.append(f" — [{m:02d}:{s:02d}]({video_url}&t={int(ts)}s)")
|
|
136
|
+
if len(atom.timestamps) > 3:
|
|
137
|
+
lines.append(f" — *+{len(atom.timestamps) - 3} more timestamps*")
|
|
138
|
+
lines.append("")
|