kd-engine 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. kd_engine-1.0.0.dist-info/METADATA +19 -0
  2. kd_engine-1.0.0.dist-info/RECORD +118 -0
  3. kd_engine-1.0.0.dist-info/WHEEL +5 -0
  4. kd_engine-1.0.0.dist-info/entry_points.txt +2 -0
  5. kd_engine-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
  6. kd_engine-1.0.0.dist-info/top_level.txt +1 -0
  7. ytk/__init__.py +2 -0
  8. ytk/__main__.py +21 -0
  9. ytk/batch/__init__.py +39 -0
  10. ytk/batch/runner.py +1149 -0
  11. ytk/cache/__init__.py +34 -0
  12. ytk/cache/v2.py +562 -0
  13. ytk/cleaning/density.py +40 -0
  14. ytk/cleaning/fillers.py +18 -0
  15. ytk/cleaning/sentence_boundary.py +30 -0
  16. ytk/cleaning/service.py +62 -0
  17. ytk/cli.py +1281 -0
  18. ytk/commands/__init__.py +3 -0
  19. ytk/commands/merge.py +600 -0
  20. ytk/config.py +757 -0
  21. ytk/dedup/service.py +166 -0
  22. ytk/doctor.py +891 -0
  23. ytk/dotenv.py +334 -0
  24. ytk/errors.py +38 -0
  25. ytk/extraction/service.py +451 -0
  26. ytk/extraction/strategies/base.py +16 -0
  27. ytk/extraction/strategies/code_block.py +285 -0
  28. ytk/extraction/strategies/heuristic.py +791 -0
  29. ytk/extraction/strategies/local_model.py +56 -0
  30. ytk/extraction/text_context.py +177 -0
  31. ytk/extraction/visual_context.py +159 -0
  32. ytk/ingestion/adapters/__init__.py +1 -0
  33. ytk/ingestion/adapters/_common.py +865 -0
  34. ytk/ingestion/adapters/base.py +160 -0
  35. ytk/ingestion/adapters/bilibili.py +271 -0
  36. ytk/ingestion/adapters/douyin.py +445 -0
  37. ytk/ingestion/adapters/tiktok.py +102 -0
  38. ytk/ingestion/adapters/xiaohongshu.py +758 -0
  39. ytk/ingestion/adapters/youtube.py +74 -0
  40. ytk/ingestion/browser_acquired.py +220 -0
  41. ytk/ingestion/http_client.py +43 -0
  42. ytk/ingestion/local_import.py +202 -0
  43. ytk/ingestion/media.py +468 -0
  44. ytk/ingestion/providers/asr.py +902 -0
  45. ytk/ingestion/providers/base.py +13 -0
  46. ytk/ingestion/providers/dtk_client.py +816 -0
  47. ytk/ingestion/providers/minimax.py +350 -0
  48. ytk/ingestion/providers/transcript_api.py +336 -0
  49. ytk/ingestion/providers/xhs_client.py +646 -0
  50. ytk/ingestion/providers/ytdlp.py +400 -0
  51. ytk/ingestion/registry.py +189 -0
  52. ytk/ingestion/resolver.py +417 -0
  53. ytk/ingestion/search.py +56 -0
  54. ytk/ingestion/service.py +167 -0
  55. ytk/ingestion/subtitles.py +602 -0
  56. ytk/ingestion/transcript_cache.py +212 -0
  57. ytk/ingestion/url_parser.py +53 -0
  58. ytk/ingestion/video_frames.py +292 -0
  59. ytk/ingestion/ytdlp_acquire.py +754 -0
  60. ytk/logging.py +383 -0
  61. ytk/models/common.py +20 -0
  62. ytk/models/content.py +451 -0
  63. ytk/models/knowledge.py +43 -0
  64. ytk/models/migrations.py +262 -0
  65. ytk/models/skill.py +13 -0
  66. ytk/models/source.py +265 -0
  67. ytk/models/topic.py +18 -0
  68. ytk/models/transcript.py +30 -0
  69. ytk/ocr/__init__.py +65 -0
  70. ytk/ocr/engine.py +245 -0
  71. ytk/ocr/layout.py +347 -0
  72. ytk/ocr/pipeline.py +440 -0
  73. ytk/ocr/rapidocr_engine.py +144 -0
  74. ytk/pipeline/acquisition_checkpoint.py +240 -0
  75. ytk/pipeline/artifacts.py +18 -0
  76. ytk/pipeline/orchestrator.py +598 -0
  77. ytk/pipeline/processing_identity.py +18 -0
  78. ytk/pipeline/retry.py +728 -0
  79. ytk/pipeline/runners.py +578 -0
  80. ytk/pipeline/state.py +1107 -0
  81. ytk/pipeline/v2.py +1863 -0
  82. ytk/py.typed +1 -0
  83. ytk/quality/__init__.py +9 -0
  84. ytk/quality/completeness.py +260 -0
  85. ytk/renderers/base.py +9 -0
  86. ytk/renderers/citation_alignment.py +617 -0
  87. ytk/renderers/evidence_formatter.py +362 -0
  88. ytk/renderers/json_renderer.py +13 -0
  89. ytk/renderers/jsonl_renderer.py +18 -0
  90. ytk/renderers/knowledge_doc_renderer.py +1984 -0
  91. ytk/renderers/markdown_renderer.py +540 -0
  92. ytk/renderers/obsidian_renderer.py +138 -0
  93. ytk/renderers/skill_package.py +21 -0
  94. ytk/renderers/source_links.py +309 -0
  95. ytk/renderers/summary_renderer.py +163 -0
  96. ytk/renderers/v2.py +171 -0
  97. ytk/renderers/v2_jsonl.py +165 -0
  98. ytk/renderers/v2_markdown.py +477 -0
  99. ytk/renderers/v2_obsidian.py +113 -0
  100. ytk/renderers/v2_summary.py +220 -0
  101. ytk/segmentation/change_points.py +50 -0
  102. ytk/segmentation/embeddings.py +45 -0
  103. ytk/segmentation/service.py +318 -0
  104. ytk/skill_builder/__init__.py +1 -0
  105. ytk/skill_builder/builder.py +419 -0
  106. ytk/skill_builder/templates/SKILL.md.j2 +85 -0
  107. ytk/skill_builder/templates/evals/evals.json.j2 +25 -0
  108. ytk/skill_builder/templates/examples/example-input.md.j2 +5 -0
  109. ytk/skill_builder/templates/examples/example-output.md.j2 +8 -0
  110. ytk/skill_builder/templates/references/commands.md.j2 +34 -0
  111. ytk/skill_builder/templates/references/concepts.md.j2 +22 -0
  112. ytk/skill_builder/templates/references/configs.md.j2 +31 -0
  113. ytk/skill_builder/templates/references/evidence-policy.md.j2 +27 -0
  114. ytk/skill_builder/templates/references/source-summary.md.j2 +24 -0
  115. ytk/skill_builder/templates/scripts/build_skill.py.j2 +14 -0
  116. ytk/skill_builder/templates/scripts/validate_output.py.j2 +136 -0
  117. ytk/skill_builder/templates.py +29 -0
  118. ytk/text_normalization.py +44 -0
@@ -0,0 +1,165 @@
1
+ """T33.2 — v2 JSONL renderer: one self-contained line per knowledge atom.
2
+
3
+ Every line carries *its own* ``sources`` / ``evidence`` / ``blocks`` /
4
+ ``assets`` / ``topics`` subset — a consumer can resolve a line without the
5
+ global bundle (contract §1.6, T33.2 acceptance: "jsonl 记录各自可解析并指向来源").
6
+ Assets use package-relative paths taken from ``MediaAsset.relative_path``; no
7
+ absolute paths and no access parameters are ever emitted.
8
+
9
+ Lines are sorted deterministically (bundle atoms are already ordered; the
10
+ object keys are sorted) so two renders produce byte-identical files.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import json
16
+ from collections.abc import Sequence
17
+ from pathlib import Path
18
+ from typing import Any
19
+
20
+ from ytk.models.content import ContentBlock, EvidenceRefV2, ExtractionBundleV2, MediaAsset
21
+ from ytk.models.source import ContentRef
22
+ from ytk.renderers.evidence_formatter import find_content, find_source, validate_evidence
23
+ from ytk.renderers.source_links import platform_link
24
+ from ytk.renderers.v2_markdown import resolve_language
25
+
26
+
27
+ def _block_payload(block: ContentBlock, source_revision: str) -> dict[str, Any]:
28
+ return {
29
+ "block_id": block.block_id,
30
+ "source_id": block.source_id,
31
+ "source_revision": source_revision,
32
+ "kind": block.kind,
33
+ "order": block.order,
34
+ "raw_text": block.raw_text,
35
+ "cleaned_text": block.cleaned_text,
36
+ "language": block.language,
37
+ "locator": block.locator.model_dump(mode="json", exclude_none=True),
38
+ }
39
+
40
+
41
+ def _asset_payload(asset: MediaAsset, source_revision: str) -> dict[str, Any]:
42
+ return {
43
+ "asset_id": asset.asset_id,
44
+ "source_id": asset.source_id,
45
+ "source_revision": source_revision,
46
+ "media_type": asset.media_type,
47
+ "relative_path": asset.relative_path,
48
+ "order": asset.order,
49
+ "byte_size": asset.byte_size,
50
+ "sha256": asset.sha256,
51
+ }
52
+
53
+
54
+ def _evidence_payload(evidence: EvidenceRefV2, ref: ContentRef, language: str) -> dict[str, Any]:
55
+ return {
56
+ "evidence_id": evidence.evidence_id,
57
+ "source_id": evidence.source_id,
58
+ "source_revision": evidence.source_revision,
59
+ "block_id": evidence.block_id,
60
+ "quote": evidence.quote,
61
+ "processor": evidence.processor,
62
+ "locator": evidence.locator.model_dump(mode="json", exclude_none=True),
63
+ "citation": platform_link(ref, evidence.locator, quote=evidence.quote, language=language),
64
+ }
65
+
66
+
67
+ def _atom_line(atom: Any, bundle: ExtractionBundleV2, language: str) -> dict[str, Any]:
68
+ sources: list[ContentRef] = []
69
+ seen_sources: set[str] = set()
70
+ for source_id in atom.source_ids:
71
+ if source_id in seen_sources:
72
+ continue
73
+ sources.append(find_source(bundle, source_id))
74
+ seen_sources.add(source_id)
75
+
76
+ evidence_payloads: list[dict[str, Any]] = []
77
+ block_payloads: list[dict[str, Any]] = []
78
+ seen_blocks: set[tuple[str, str, str]] = set()
79
+ block_ids: list[str] = []
80
+ for evidence_id in atom.evidence_ids:
81
+ evidence = bundle.evidence_index.get(evidence_id)
82
+ if evidence is None: # referenced atoms are validated by the bundle itself
83
+ raise ValueError(
84
+ f"atom {atom.atom_id} references evidence {evidence_id!r} missing from the index"
85
+ )
86
+ validate_evidence(evidence, bundle)
87
+ ref = find_source(bundle, evidence.source_id)
88
+ evidence_payloads.append(_evidence_payload(evidence, ref, language))
89
+ key = (evidence.source_id, evidence.source_revision, evidence.block_id)
90
+ if key in seen_blocks:
91
+ continue
92
+ seen_blocks.add(key)
93
+ content = find_content(bundle, evidence.source_id, evidence.source_revision)
94
+ block = next(
95
+ (candidate for candidate in content.blocks if candidate.block_id == evidence.block_id),
96
+ None,
97
+ )
98
+ if block is not None:
99
+ block_payloads.append(_block_payload(block, evidence.source_revision))
100
+ block_ids.append(block.block_id)
101
+
102
+ assets: list[dict[str, Any]] = []
103
+ seen_assets: set[tuple[str, str, str]] = set()
104
+ for source_id in [source.source_id for source in sources]:
105
+ for content in bundle.contents:
106
+ if content.source.source_id != source_id:
107
+ continue
108
+ for asset in content.assets:
109
+ asset_key = (asset.source_id, content.source_revision, asset.asset_id)
110
+ if asset_key in seen_assets:
111
+ continue
112
+ seen_assets.add(asset_key)
113
+ assets.append(_asset_payload(asset, content.source_revision))
114
+
115
+ topics: list[dict[str, str]] = []
116
+ for topic in bundle.topics:
117
+ if any(block_id in topic.block_ids for block_id in block_ids):
118
+ topics.append({"topic_id": topic.topic_id, "title": topic.title})
119
+
120
+ return {
121
+ "schema_version": 2,
122
+ "atom_id": atom.atom_id,
123
+ "atom_type": atom.atom_type,
124
+ "content": atom.content,
125
+ "explanation": atom.explanation,
126
+ "confidence": atom.confidence,
127
+ "topic_path": list(atom.topic_path),
128
+ "source_ids": list(atom.source_ids),
129
+ "sources": [ref.public_dict() for ref in sources],
130
+ "evidence": evidence_payloads,
131
+ "blocks": block_payloads,
132
+ "assets": assets,
133
+ "topics": topics,
134
+ }
135
+
136
+
137
+ class V2JSONLRenderer:
138
+ """Renderer-protocol compatible v2 JSONL renderer."""
139
+
140
+ def render(self, bundle: ExtractionBundleV2, output_path: Path, **kwargs: Any) -> None:
141
+ language = resolve_language(kwargs)
142
+ output_path = Path(output_path)
143
+ output_path.parent.mkdir(parents=True, exist_ok=True)
144
+ lines = [
145
+ json.dumps(_atom_line(atom, bundle, language), ensure_ascii=False, sort_keys=True)
146
+ for atom in bundle.knowledge
147
+ ]
148
+ output_path.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8")
149
+
150
+
151
+ def render_jsonl(bundle: ExtractionBundleV2, output_path: Path, **kwargs: Any) -> None:
152
+ V2JSONLRenderer().render(bundle, output_path, **kwargs)
153
+
154
+
155
+ def resolve_jsonl_references(line: str | dict[str, Any]) -> Sequence[str]:
156
+ """Return the evidence ids a rendered line points at (diagnostics helper)."""
157
+ payload = json.loads(line) if isinstance(line, str) else line
158
+ return [entry["evidence_id"] for entry in payload.get("evidence", [])]
159
+
160
+
161
+ __all__ = [
162
+ "V2JSONLRenderer",
163
+ "render_jsonl",
164
+ "resolve_jsonl_references",
165
+ ]
@@ -0,0 +1,477 @@
1
+ """T33.2 — v2 Markdown notes renderer (``notes.md``).
2
+
3
+ The legacy :class:`~ytk.renderers.markdown_renderer.MarkdownRenderer` keeps
4
+ rendering v1 bundles byte-for-byte; this module renders the v2 model
5
+ (:class:`~ytk.models.content.ExtractionBundleV2`) without touching it.
6
+
7
+ Rules inherited from the frozen v2 contract (§1.6):
8
+
9
+ * every source is a :class:`ContentRef` — the provenance block is built with
10
+ :func:`ytk.renderers.source_links.source_bullet` (five platforms, no
11
+ hard-coded YouTube link);
12
+ * every citation comes from
13
+ :func:`ytk.renderers.evidence_formatter.format_evidence_refs`, which also
14
+ validates it against the bundle first (a fabricated citation raises instead
15
+ of landing in the note);
16
+ * image notes reference their images with package-relative paths from
17
+ :class:`~ytk.models.content.MediaAsset` and never emit ``[00:00]``-style
18
+ timestamps;
19
+ * output is deterministic: no clock reads, no locale probes, no network/LLM.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ from collections.abc import Iterable
25
+ from pathlib import Path
26
+ from typing import Any
27
+
28
+ from ytk.models.content import (
29
+ ContentBlock,
30
+ ContentBundle,
31
+ ExtractionBundleV2,
32
+ ImageRegionLocator,
33
+ KnowledgeAtomV2,
34
+ ParagraphLocator,
35
+ TimeRangeLocator,
36
+ VideoFrameLocator,
37
+ )
38
+ from ytk.models.source import ContentRef
39
+ from ytk.extraction.service import AUTHOR_TEXT_PROCESSORS
40
+ from ytk.renderers.evidence_formatter import (
41
+ evidence_for_atom,
42
+ find_content,
43
+ )
44
+ from ytk.renderers.source_links import (
45
+ display_title,
46
+ image_markdown_ref,
47
+ platform_label,
48
+ source_bullet,
49
+ platform_link,
50
+ )
51
+
52
+ ATOM_TYPE_TITLES: dict[str, dict[str, str]] = {
53
+ "terminal_command": {"zh": "命令", "en": "Commands"},
54
+ "configuration_snippet": {"zh": "配置", "en": "Configuration"},
55
+ "workflow_step": {"zh": "工作流程", "en": "Workflow Steps"},
56
+ "best_practice": {"zh": "最佳实践", "en": "Best Practices"},
57
+ "core_concept": {"zh": "核心概念", "en": "Core Concepts"},
58
+ "code_block": {"zh": "代码块", "en": "Code Blocks"},
59
+ "command_sequence": {"zh": "命令序列", "en": "Command Sequences"},
60
+ "insight": {"zh": "洞察", "en": "Insights"},
61
+ "reference": {"zh": "参考", "en": "References"},
62
+ "segment_summary": {"zh": "分段摘要(非知识结论)", "en": "Segment excerpt (not a knowledge conclusion)"},
63
+ }
64
+
65
+ _ATOM_TYPE_ORDER: tuple[str, ...] = tuple(ATOM_TYPE_TITLES)
66
+
67
+ _UI: dict[str, dict[str, str]] = {
68
+ "zh": {
69
+ "sources": "## 来源",
70
+ "overview": "## 概述",
71
+ "topics": "## 主题",
72
+ "ungrouped": "## 未分组",
73
+ "images": "## 图片",
74
+ "stats": "## 统计",
75
+ "no_atoms": "> ⚠️ 未提取到可靠知识点。",
76
+ "confidence": "置信度",
77
+ "explanation": "说明",
78
+ "evidence": "证据",
79
+ "atom_count": "知识点总数",
80
+ "source_count": "来源数",
81
+ "content_type": "内容类型",
82
+ "status": "内容完整性",
83
+ "generated": "*由 knowledge-distiller v2.0 生成*",
84
+ "image": "图",
85
+ "no_topics": "未分组知识点",
86
+ "author_text_evidence": "来自标题/简介",
87
+ },
88
+ "en": {
89
+ "sources": "## Sources",
90
+ "overview": "## Overview",
91
+ "topics": "## Topics",
92
+ "ungrouped": "## Ungrouped",
93
+ "images": "## Images",
94
+ "stats": "## Stats",
95
+ "no_atoms": "> ⚠️ No reliable knowledge atoms could be extracted.",
96
+ "confidence": "Confidence",
97
+ "explanation": "Explanation",
98
+ "evidence": "Evidence",
99
+ "atom_count": "Total knowledge atoms",
100
+ "source_count": "Sources",
101
+ "content_type": "Content type",
102
+ "status": "Completeness",
103
+ "generated": "*Generated by knowledge-distiller v2.0*",
104
+ "image": "Image",
105
+ "no_topics": "Ungrouped knowledge",
106
+ "author_text_evidence": "from title/description",
107
+ },
108
+ }
109
+
110
+ CONTENT_TYPE_LABELS: dict[str, dict[str, str]] = {
111
+ "video": {"zh": "视频", "en": "video"},
112
+ "image_note": {"zh": "图文笔记", "en": "image note"},
113
+ }
114
+
115
+
116
+ def resolve_language(kwargs: dict[str, Any] | None = None) -> str:
117
+ """Resolve the output language from renderer kwargs (default ``zh``)."""
118
+ payload = kwargs or {}
119
+ for key in ("language", "doc_language"):
120
+ value = payload.get(key)
121
+ if isinstance(value, str) and value.strip().lower() == "en":
122
+ return "en"
123
+ if isinstance(value, str) and value.strip().lower() == "zh":
124
+ return "zh"
125
+ return "zh"
126
+
127
+
128
+ def ui(language: str, key: str) -> str:
129
+ return _UI["en" if language == "en" else "zh"][key]
130
+
131
+
132
+ def atom_type_title(atom_type: str, language: str) -> str:
133
+ labels = ATOM_TYPE_TITLES.get(atom_type)
134
+ if labels is None:
135
+ return atom_type.replace("_", " ").title()
136
+ return labels["en" if language == "en" else "zh"]
137
+
138
+
139
+ def bundle_title(bundle: ExtractionBundleV2) -> str:
140
+ """Deterministic title: primary source display title, then metadata."""
141
+ for key in ("title", "display_title"):
142
+ value = bundle.metadata.get(key)
143
+ if isinstance(value, str) and value.strip():
144
+ return value.strip()
145
+ if bundle.sources:
146
+ return display_title(bundle.sources[0])
147
+ fallback = bundle.metadata.get("source_id")
148
+ if isinstance(fallback, str) and fallback.strip():
149
+ return fallback.strip()
150
+ return "Knowledge"
151
+
152
+
153
+ def ordered_sources(bundle: ExtractionBundleV2) -> list[ContentRef]:
154
+ """Every registered source in bundle order (deterministic)."""
155
+ return list(bundle.sources)
156
+
157
+
158
+ def atoms_by_topic(
159
+ bundle: ExtractionBundleV2,
160
+ ) -> tuple[dict[str, list[KnowledgeAtomV2]], list[KnowledgeAtomV2]]:
161
+ """Group atoms by the topic their evidence blocks belong to.
162
+
163
+ Returns ``(topic_id -> atoms, ungrouped atoms)``. An atom belongs to the
164
+ first topic (bundle order) that contains one of its evidence blocks, which
165
+ keeps the mapping stable for repeated renders.
166
+ """
167
+ topic_by_block: dict[str, str] = {}
168
+ for topic in bundle.topics:
169
+ for block_id in topic.block_ids:
170
+ topic_by_block.setdefault(block_id, topic.topic_id)
171
+ grouped: dict[str, list[KnowledgeAtomV2]] = {}
172
+ ungrouped: list[KnowledgeAtomV2] = []
173
+ for atom in bundle.knowledge:
174
+ topic_id: str | None = None
175
+ for evidence_id in atom.evidence_ids:
176
+ evidence = bundle.evidence_index.get(evidence_id)
177
+ if evidence is None:
178
+ continue
179
+ candidate = topic_by_block.get(evidence.block_id)
180
+ if candidate is not None:
181
+ topic_id = candidate
182
+ break
183
+ if topic_id is None:
184
+ ungrouped.append(atom)
185
+ else:
186
+ grouped.setdefault(topic_id, []).append(atom)
187
+ return grouped, ungrouped
188
+
189
+
190
+ def atoms_of_source(bundle: ExtractionBundleV2, source_id: str) -> list[KnowledgeAtomV2]:
191
+ return [atom for atom in bundle.knowledge if source_id in atom.source_ids]
192
+
193
+
194
+ def image_assets(content: ContentBundle) -> list[Any]:
195
+ """Image assets in their recorded order (1-based image index order)."""
196
+ images = [asset for asset in content.assets if asset.media_type == "image"]
197
+ return sorted(images, key=lambda asset: (asset.order, asset.asset_id))
198
+
199
+
200
+ def content_blocks_by_id(content: ContentBundle) -> dict[str, ContentBlock]:
201
+ return {block.block_id: block for block in content.blocks}
202
+
203
+
204
+ def evidence_blocks(atom: KnowledgeAtomV2, bundle: ExtractionBundleV2) -> list[ContentBlock]:
205
+ """The distinct content blocks an atom's evidence points at (bundle order)."""
206
+ blocks: list[ContentBlock] = []
207
+ seen: set[str] = set()
208
+ for evidence_id in atom.evidence_ids:
209
+ evidence = bundle.evidence_index.get(evidence_id)
210
+ if evidence is None or evidence.block_id in seen:
211
+ continue
212
+ seen.add(evidence.block_id)
213
+ try:
214
+ content = find_content(bundle, evidence.source_id, evidence.source_revision)
215
+ except Exception: # pragma: no cover - validated upstream for real renders
216
+ continue
217
+ block = content_blocks_by_id(content).get(evidence.block_id)
218
+ if block is not None:
219
+ blocks.append(block)
220
+ return blocks
221
+
222
+
223
+ def render_sources_block(bundle: ExtractionBundleV2, language: str) -> list[str]:
224
+ lines = [ui(language, "sources"), ""]
225
+ for ref in ordered_sources(bundle):
226
+ lines.append(source_bullet(ref, language=language))
227
+ content_type = CONTENT_TYPE_LABELS.get(
228
+ bundle.sources[0].content_type if bundle.sources else "video",
229
+ CONTENT_TYPE_LABELS["video"],
230
+ )
231
+ type_label = content_type["en" if language == "en" else "zh"]
232
+ lines.append("")
233
+ statuses = sorted({content.completeness.status for content in bundle.contents})
234
+ if statuses:
235
+ lines.append(f"- {ui(language, 'content_type')}: {type_label}")
236
+ lines.append(f"- {ui(language, 'status')}: {', '.join(statuses)}")
237
+ lines.append("")
238
+ return lines
239
+
240
+
241
+ def render_processing_notes(bundle: ExtractionBundleV2, language: str) -> list[str]:
242
+ """State what was actually acquired; full captions do not prove visual coverage."""
243
+ en = language == "en"
244
+ lines = ["## Processing scope" if en else "## 处理范围与限制", ""]
245
+ for content in bundle.contents:
246
+ metadata = content.metadata
247
+ coverage = metadata.get("coverage")
248
+ coverage = coverage if isinstance(coverage, dict) else {}
249
+ processors = {ev.processor for ev in bundle.evidence_index.values()
250
+ if ev.source_id == content.source.source_id and ev.source_revision == content.source_revision}
251
+ transcript = metadata.get("transcript_source")
252
+ notes = []
253
+ if metadata.get("acquisition_mode") == "user_import":
254
+ notes.append("User-provided file; its pairing with the original source is not verified"
255
+ if en else "用户提供的文件;与原始来源的对应关系未验证")
256
+ if transcript == "auto" or "subtitle_auto" in processors:
257
+ notes.append("Automatic captions, not human reviewed" if en else "自动字幕,未经人工校对")
258
+ elif transcript == "asr" or "asr" in processors:
259
+ notes.append("ASR transcript, not human reviewed" if en else "语音识别转写,未经人工校对")
260
+ notes.append("Extraction scores do not measure transcription accuracy" if en else "提炼分数不代表转写准确率")
261
+ elif transcript == "user_provided":
262
+ notes.append("User-provided subtitles; accuracy is not independently verified"
263
+ if en else "用户提供的字幕,准确性未独立验证")
264
+ if content.source.content_type == "video":
265
+ has_frames = any(isinstance(block.locator, VideoFrameLocator) for block in content.blocks)
266
+ if has_frames and coverage.get("visuals") != "not_analyzed":
267
+ notes.append("Only acquired frame evidence was analysed; complete visual coverage is unverified"
268
+ if en else "仅处理已采集的画面证据,未验证完整画面覆盖")
269
+ else:
270
+ notes.append("Visuals not analysed; slides, demonstrations and on-screen text may be missing"
271
+ if en else "未分析画面;幻灯片、操作演示与屏幕文字可能缺失")
272
+ if content.completeness.status != "complete":
273
+ missing = ", ".join(content.completeness.missing)
274
+ notes.append(("Acquisition is " if en else "采集为 ") + content.completeness.status + (f"; missing: {missing}" if missing else ""))
275
+ if notes:
276
+ lines.append(f"- {platform_label(content.source)} · {display_title(content.source)}: " + "; ".join(notes))
277
+ if any(atom.atom_type == "segment_summary" for atom in bundle.knowledge):
278
+ lines.append("> Segment excerpts are source indexes, not knowledge conclusions."
279
+ if en else "> 分段摘要只用于索引原文,属于非知识结论,不可当作独立证据确认主张。")
280
+ if len(lines) == 2:
281
+ return []
282
+ return lines + [""]
283
+
284
+
285
+ def render_source_index(bundle: ExtractionBundleV2, language: str) -> list[str]:
286
+ """Useful, explicitly non-knowledge output when no structured atoms qualify."""
287
+ en = language == "en"
288
+ entries = [(content, block) for content in bundle.contents for block in content.blocks
289
+ if (isinstance(block.locator, TimeRangeLocator) and block.kind in {"transcript", "caption"})
290
+ or isinstance(block.locator, VideoFrameLocator)]
291
+ if not entries:
292
+ return []
293
+ has_frames = any(isinstance(block.locator, VideoFrameLocator) for _, block in entries)
294
+ heading = ("## Timeline and sampled observation index (not a knowledge conclusion)" if en else "## 时间轴与抽样观察索引(非知识结论)") if has_frames else ("## Timeline index (not a knowledge conclusion)" if en else "## 时间轴索引(非知识结论)")
295
+ lines = [heading, ""]
296
+ for content, block in entries[:40]:
297
+ lines.append("- " + platform_link(content.source, block.locator, quote=block.raw_text, language=language))
298
+ if len(entries) > 40:
299
+ lines.append(f"\n> Showing the first 40 of {len(entries)} source cues; the complete cues remain in knowledge.json."
300
+ if en else f"\n> 这里只展示前40条,共{len(entries)}条原始片段;完整片段保留在 knowledge.json 中。")
301
+ return lines + [""]
302
+
303
+
304
+ def render_image_sections(bundle: ExtractionBundleV2, language: str) -> list[str]:
305
+ """Image-note assets as portable relative-path markdown image refs."""
306
+ lines: list[str] = []
307
+ for content in bundle.contents:
308
+ if content.source.content_type != "image_note":
309
+ continue
310
+ if not image_assets(content):
311
+ continue
312
+ lines.append(ui(language, "images"))
313
+ lines.append("")
314
+ if len(bundle.contents) > 1:
315
+ lines.append(f"### {platform_label(content.source)} · {display_title(content.source)}")
316
+ lines.append("")
317
+ for index, asset in enumerate(image_assets(content), start=1):
318
+ alt = f"{ui(language, 'image')} {index}"
319
+ lines.append(image_markdown_ref(asset.relative_path, alt=alt))
320
+ lines.append("")
321
+ return lines
322
+
323
+
324
+ def author_text_evidence_ids(atom: KnowledgeAtomV2, bundle: ExtractionBundleV2) -> set[str]:
325
+ """Evidence refs of ``atom`` that come from the author's own text.
326
+
327
+ Title/description evidence is not a transcript, so the rendered evidence
328
+ line has to say where it came from (U2, 2026-11-14).
329
+ """
330
+ found: set[str] = set()
331
+ for evidence_id in atom.evidence_ids:
332
+ evidence = bundle.evidence_index.get(evidence_id)
333
+ if evidence is not None and evidence.processor in AUTHOR_TEXT_PROCESSORS:
334
+ found.add(evidence_id)
335
+ return found
336
+
337
+
338
+ def render_atom(atom: KnowledgeAtomV2, bundle: ExtractionBundleV2, language: str) -> list[str]:
339
+ lines = [f"### {atom_type_title(atom.atom_type, language)}: {atom.content}", ""]
340
+ lines.append(f"- {ui(language, 'confidence')}: {atom.confidence:.2f}")
341
+ if atom.explanation:
342
+ lines.append(f"- {ui(language, 'explanation')}: {atom.explanation}")
343
+ author_ids = author_text_evidence_ids(atom, bundle)
344
+ for evidence_id, rendered in zip(atom.evidence_ids,
345
+ evidence_for_atom(atom, bundle, language=language),
346
+ strict=False):
347
+ origin = ui(language, "author_text_evidence")
348
+ prefix = f"{origin} · " if evidence_id in author_ids else ""
349
+ lines.append(f"- {ui(language, 'evidence')}: {prefix}{rendered}")
350
+ lines.append("")
351
+ return lines
352
+
353
+
354
+ def render_body(bundle: ExtractionBundleV2, language: str) -> list[str]:
355
+ """The shared Markdown body (used by both ``notes.md`` and Obsidian)."""
356
+ lines: list[str] = [f"# {bundle_title(bundle)}", ""]
357
+ lines.extend(render_sources_block(bundle, language))
358
+ lines.extend(render_processing_notes(bundle, language))
359
+ has_asr = any(content.metadata.get("transcript_source") == "asr" for content in bundle.contents) or any(
360
+ evidence.processor == "asr" for evidence in bundle.evidence_index.values()
361
+ )
362
+ overview = ("## ASR topic index (requires review)" if language == "en" else "## 语音转写主题索引(待核对)") if has_asr else ui(language, "overview")
363
+ lines.extend([overview, ""])
364
+
365
+ grouped, ungrouped = atoms_by_topic(bundle)
366
+ if not bundle.knowledge:
367
+ lines.extend([ui(language, "no_atoms"), ""])
368
+ lines.extend(render_source_index(bundle, language))
369
+ elif bundle.topics:
370
+ for topic in bundle.topics:
371
+ members = grouped.get(topic.topic_id, [])
372
+ bullet = f"- {topic.title}"
373
+ if members:
374
+ bullet += f" ({len(members)})"
375
+ lines.append(bullet)
376
+ lines.append("")
377
+ else:
378
+ lines.append(ui(language, "no_topics"))
379
+ lines.append("")
380
+
381
+ ordered_groups: list[tuple[str, str, list[KnowledgeAtomV2]]] = []
382
+ if bundle.topics:
383
+ topic_titles = {topic.topic_id: topic.title for topic in bundle.topics}
384
+ for topic in bundle.topics:
385
+ members = grouped.get(topic.topic_id, [])
386
+ if members:
387
+ ordered_groups.append((topic.topic_id, topic_titles[topic.topic_id], members))
388
+ if ungrouped:
389
+ ordered_groups.append(("__ungrouped__", ui(language, "ungrouped"), ungrouped))
390
+
391
+ for _topic_id, title, members in ordered_groups:
392
+ prefix = ("Transcript topic: " if language == "en" else "原转写主题:") if has_asr else ""
393
+ heading = f"## {prefix}{title}" if _topic_id != "__ungrouped__" else title
394
+ lines.extend([heading, ""])
395
+ for atom in members:
396
+ lines.extend(render_atom(atom, bundle, language))
397
+
398
+ lines.extend(render_image_sections(bundle, language))
399
+
400
+ lines.extend([ui(language, "stats"), ""])
401
+ lines.append(f"- {ui(language, 'source_count')}: {len(bundle.sources)}")
402
+ lines.append(f"- {ui(language, 'atom_count')}: {len(bundle.knowledge)}")
403
+ counts: dict[str, int] = {}
404
+ for atom in bundle.knowledge:
405
+ counts[atom.atom_type] = counts.get(atom.atom_type, 0) + 1
406
+ for atom_type in _ATOM_TYPE_ORDER:
407
+ if counts.get(atom_type):
408
+ lines.append(f"- {atom_type_title(atom_type, language)}: {counts[atom_type]}")
409
+ for atom_type in sorted(counts):
410
+ if atom_type not in _ATOM_TYPE_ORDER:
411
+ lines.append(f"- {atom_type_title(atom_type, language)}: {counts[atom_type]}")
412
+ lines.extend(["", ui(language, "generated"), ""])
413
+ return lines
414
+
415
+
416
+ class V2MarkdownRenderer:
417
+ """Renderer-protocol compatible v2 Markdown renderer."""
418
+
419
+ def render(self, bundle: ExtractionBundleV2, output_path: Path, **kwargs: Any) -> None:
420
+ language = resolve_language(kwargs)
421
+ output_path = Path(output_path)
422
+ output_path.parent.mkdir(parents=True, exist_ok=True)
423
+ lines = render_body(bundle, language)
424
+ output_path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
425
+
426
+
427
+ def render_markdown(bundle: ExtractionBundleV2, output_path: Path, **kwargs: Any) -> None:
428
+ V2MarkdownRenderer().render(bundle, output_path, **kwargs)
429
+
430
+
431
+ def sort_atoms(atoms: Iterable[KnowledgeAtomV2]) -> list[KnowledgeAtomV2]:
432
+ """Deterministic atom order: type order, then lowercase content."""
433
+ return sorted(
434
+ atoms,
435
+ key=lambda atom: (
436
+ _ATOM_TYPE_ORDER.index(atom.atom_type)
437
+ if atom.atom_type in _ATOM_TYPE_ORDER
438
+ else len(_ATOM_TYPE_ORDER),
439
+ atom.content.casefold(),
440
+ atom.atom_id,
441
+ ),
442
+ )
443
+
444
+
445
+ def locator_kind_of(block: ContentBlock) -> str:
446
+ locator = block.locator
447
+ if isinstance(locator, TimeRangeLocator):
448
+ return "time_range"
449
+ if isinstance(locator, VideoFrameLocator):
450
+ return "video_frame"
451
+ if isinstance(locator, ImageRegionLocator):
452
+ return "image_region"
453
+ if isinstance(locator, ParagraphLocator):
454
+ return "paragraph"
455
+ return locator.kind
456
+
457
+
458
+ __all__ = [
459
+ "ATOM_TYPE_TITLES",
460
+ "CONTENT_TYPE_LABELS",
461
+ "V2MarkdownRenderer",
462
+ "atom_type_title",
463
+ "atoms_by_topic",
464
+ "atoms_of_source",
465
+ "bundle_title",
466
+ "content_blocks_by_id",
467
+ "evidence_blocks",
468
+ "image_assets",
469
+ "ordered_sources",
470
+ "render_body",
471
+ "render_image_sections",
472
+ "render_markdown",
473
+ "render_sources_block",
474
+ "resolve_language",
475
+ "sort_atoms",
476
+ "ui",
477
+ ]